Skip to main content

iphone_cli/ax/
tree.rs

1//! Turn a raw dump into the element list `iph read` prints.
2
3use std::collections::HashSet;
4
5use serde::{Deserialize, Serialize};
6
7use super::{
8    archive::Ka,
9    attrs::{self, traits},
10    dump::{Dump, ObjKey, RawElement},
11};
12
13#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
14pub struct Rect {
15    pub x: f64,
16    pub y: f64,
17    pub w: f64,
18    pub h: f64,
19}
20
21impl Rect {
22    pub fn center(&self) -> (f64, f64) {
23        (self.x + self.w / 2.0, self.y + self.h / 2.0)
24    }
25
26    pub fn contains(&self, x: f64, y: f64) -> bool {
27        x >= self.x && x <= self.x + self.w && y >= self.y && y <= self.y + self.h
28    }
29
30    pub fn area(&self) -> f64 {
31        self.w.max(0.0) * self.h.max(0.0)
32    }
33
34    pub fn intersection(&self, other: &Rect) -> Option<Rect> {
35        let x1 = self.x.max(other.x);
36        let y1 = self.y.max(other.y);
37        let x2 = (self.x + self.w).min(other.x + other.w);
38        let y2 = (self.y + self.h).min(other.y + other.h);
39        (x2 > x1 && y2 > y1).then_some(Rect {
40            x: x1,
41            y: y1,
42            w: x2 - x1,
43            h: y2 - y1,
44        })
45    }
46
47    /// Parse `{{x, y}, {w, h}}`.
48    pub fn parse(s: &str) -> Option<Rect> {
49        let nums: Vec<f64> = s
50            .split(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace())
51            .filter(|p| !p.is_empty())
52            .map(|p| p.parse::<f64>())
53            .collect::<Result<_, _>>()
54            .ok()?;
55        if nums.len() != 4 || nums.iter().any(|n| !n.is_finite()) {
56            return None;
57        }
58        Some(Rect {
59            x: round1(nums[0]),
60            y: round1(nums[1]),
61            w: round1(nums[2]),
62            h: round1(nums[3]),
63        })
64    }
65}
66
67fn round1(v: f64) -> f64 {
68    (v * 10.0).round() / 10.0
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
72pub struct Element {
73    /// Short handle for this element, stable while the element lives.
74    pub id: String,
75    pub role: String,
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub label: Option<String>,
78    #[serde(skip_serializing_if = "Option::is_none")]
79    pub value: Option<String>,
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub identifier: Option<String>,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub hint: Option<String>,
84    #[serde(skip_serializing_if = "Option::is_none")]
85    pub frame: Option<Rect>,
86    #[serde(default)]
87    pub traits: Vec<String>,
88    #[serde(default = "yes")]
89    pub enabled: bool,
90    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
91    pub selected: bool,
92    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
93    pub focused: bool,
94    /// `id` of the parent element, absent for an application.
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub parent: Option<String>,
97    #[serde(default)]
98    pub depth: u32,
99    #[serde(default)]
100    pub pid: i64,
101    /// False when the element lies outside the screen.
102    #[serde(default = "yes")]
103    pub on_screen: bool,
104    /// Every numbered attribute, only with `--raw`.
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub raw: Option<serde_json::Value>,
107    /// The phone's name for the element, to ask it about one attribute.
108    #[serde(skip)]
109    pub key: Option<ObjKey>,
110}
111
112fn yes() -> bool {
113    true
114}
115
116#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
117pub struct AppInfo {
118    pub pid: i64,
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub name: Option<String>,
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub bundle_id: Option<String>,
123}
124
125#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
126pub struct Screen {
127    /// Logical screen size in points.
128    pub width: f64,
129    pub height: f64,
130    pub apps: Vec<AppInfo>,
131    pub elements: Vec<Element>,
132}
133
134/// Role, label, value and rounded frame of one element.
135pub type ElementSignature = (String, Option<String>, Option<String>, Option<(i64, i64, i64, i64)>);
136
137#[derive(Debug, Clone, Copy, Default)]
138pub struct BuildOptions {
139    /// Keep elements that lie outside the screen and walk hidden children.
140    pub include_offscreen: bool,
141    /// Attach every numbered attribute to each element.
142    pub raw: bool,
143}
144
145fn handle(key: &ObjKey) -> String {
146    // Fold the 64-bit object id and the pid into 32 bits; unique enough within
147    // one screen, and `Screen::build` de-duplicates the rare collision.
148    let mut h = key.object_id ^ (key.object_id >> 32) ^ ((key.pid as u64) << 17);
149    h = h.wrapping_mul(0x9E37_79B9_7F4A_7C15);
150    format!("{:08x}", (h >> 32) as u32)
151}
152
153pub(crate) fn text(v: Option<&Ka>) -> Option<String> {
154    let s = match v? {
155        Ka::Str(s) => s.clone(),
156        other @ Ka::Obj { .. } => other.text()?,
157        Ka::Int(i) => i.to_string(),
158        Ka::UInt(u) => u.to_string(),
159        Ka::Real(r) => r.to_string(),
160        _ => return None,
161    };
162    let t = s.trim();
163    (!t.is_empty()).then(|| t.to_string())
164}
165
166fn frame_of(raw: &RawElement) -> Option<Rect> {
167    raw.attrs
168        .get(&attrs::FRAME)
169        .and_then(|v| v.get("NS.rectval"))
170        .and_then(|v| v.as_str())
171        .and_then(Rect::parse)
172}
173
174pub(crate) fn children_of(raw: &RawElement, include_offscreen: bool) -> Vec<ObjKey> {
175    let pick = |n: i64| -> Vec<ObjKey> {
176        raw.attrs
177            .get(&n)
178            .and_then(|v| v.as_array())
179            .map(|a| a.iter().filter_map(ObjKey::from_ka).collect())
180            .unwrap_or_default()
181    };
182    let (first, second) = if include_offscreen {
183        (attrs::CHILDREN, attrs::VISIBLE_CHILDREN)
184    } else {
185        (attrs::VISIBLE_CHILDREN, attrs::CHILDREN)
186    };
187    let list = pick(first);
188    if list.is_empty() { pick(second) } else { list }
189}
190
191impl Screen {
192    pub fn build(dump: &Dump, opts: BuildOptions) -> Screen {
193        let mut screen = Screen {
194            width: 0.0,
195            height: 0.0,
196            apps: Vec::new(),
197            elements: Vec::new(),
198        };
199        // The screen is as large as the largest application frame.
200        for key in &dump.applications {
201            if let Some(f) = dump.elements.get(key).and_then(frame_of) {
202                screen.width = screen.width.max(f.x + f.w);
203                screen.height = screen.height.max(f.y + f.h);
204            }
205        }
206        let bounds = Rect {
207            x: 0.0,
208            y: 0.0,
209            w: screen.width,
210            h: screen.height,
211        };
212
213        let mut seen: HashSet<ObjKey> = HashSet::new();
214        let mut used_ids: HashSet<String> = HashSet::new();
215        // Depth-first from each application, in the phone's own child order.
216        let mut stack: Vec<(ObjKey, Option<String>, u32)> = dump
217            .applications
218            .iter()
219            .rev()
220            .map(|k| (*k, None, 0))
221            .collect();
222        // Elements the phone described but no application reaches (rare).
223        let mut orphans: Vec<ObjKey> = Vec::new();
224
225        loop {
226            let Some((key, parent, depth)) = stack.pop() else {
227                if orphans.is_empty() {
228                    orphans = dump
229                        .order
230                        .iter()
231                        .filter(|k| !seen.contains(k))
232                        .copied()
233                        .collect();
234                    if orphans.is_empty() {
235                        break;
236                    }
237                    for k in orphans.iter().rev() {
238                        stack.push((*k, None, 1));
239                    }
240                    continue;
241                }
242                break;
243            };
244            if !seen.insert(key) {
245                continue;
246            }
247            let Some(raw) = dump.elements.get(&key) else {
248                continue;
249            };
250            let bits = raw
251                .attrs
252                .get(&attrs::TRAITS)
253                .and_then(|v| v.as_u64())
254                .unwrap_or(0);
255            let role_num = raw.attrs.get(&attrs::ROLE).and_then(|v| v.as_i64());
256            let frame = frame_of(raw);
257            let on_screen = match (&frame, bounds.area() > 0.0) {
258                (Some(f), true) => f.intersection(&bounds).is_some(),
259                (Some(f), false) => f.area() > 0.0,
260                (None, _) => raw.is_application,
261            };
262
263            let mut id = handle(&key);
264            while !used_ids.insert(id.clone()) {
265                id.push('x');
266            }
267            for child in children_of(raw, opts.include_offscreen).into_iter().rev() {
268                stack.push((child, Some(id.clone()), depth + 1));
269            }
270            if raw.is_application {
271                screen.apps.push(AppInfo {
272                    pid: key.pid,
273                    name: text(raw.attrs.get(&attrs::LABEL))
274                        .or_else(|| text(raw.attrs.get(&attrs::LABEL_PLAIN))),
275                    bundle_id: None,
276                });
277            }
278            if !on_screen && !opts.include_offscreen && !raw.is_application {
279                continue;
280            }
281            let label = text(raw.attrs.get(&attrs::LABEL))
282                .or_else(|| text(raw.attrs.get(&attrs::LABEL_PLAIN)));
283            let value = text(raw.attrs.get(&attrs::VALUE));
284            let has_text = label.is_some() || value.is_some();
285            screen.elements.push(Element {
286                id,
287                role: attrs::role_name(role_num, bits, raw.is_application, has_text).to_string(),
288                label,
289                value,
290                identifier: text(raw.attrs.get(&attrs::IDENTIFIER)),
291                hint: text(raw.attrs.get(&attrs::HINT)),
292                frame,
293                traits: attrs::trait_names(bits)
294                    .into_iter()
295                    .map(String::from)
296                    .collect(),
297                enabled: bits & traits::NOT_ENABLED == 0,
298                selected: bits & traits::SELECTED != 0,
299                focused: bits & traits::IS_EDITING != 0,
300                parent,
301                depth,
302                pid: key.pid,
303                key: Some(key),
304                on_screen,
305                raw: opts.raw.then(|| {
306                    let mut m = serde_json::Map::new();
307                    for (n, v) in &raw.attrs {
308                        m.insert(n.to_string(), v.to_json());
309                    }
310                    for (n, v) in &raw.singles {
311                        m.insert(format!("single_{n}"), v.to_json());
312                    }
313                    serde_json::Value::Object(m)
314                }),
315            });
316        }
317        screen
318    }
319
320    /// What a comparison of two readings looks at: everything a user could
321    /// see or an agent could act on, without handles.
322    pub fn signature(&self) -> Vec<ElementSignature> {
323        self.elements
324            .iter()
325            .map(|e| {
326                (
327                    e.role.clone(),
328                    e.label.clone(),
329                    e.value.clone(),
330                    e.frame.map(|f| {
331                        (
332                            f.x.round() as i64,
333                            f.y.round() as i64,
334                            f.w.round() as i64,
335                            f.h.round() as i64,
336                        )
337                    }),
338                )
339            })
340            .collect()
341    }
342
343    pub fn by_id(&self, id: &str) -> Option<&Element> {
344        self.elements.iter().find(|e| e.id == id)
345    }
346
347    /// The smallest on-screen element whose frame contains the point.
348    pub fn element_at(&self, x: f64, y: f64) -> Option<&Element> {
349        self.elements
350            .iter()
351            .filter(|e| e.role != "application")
352            .filter_map(|e| e.frame.map(|f| (e, f)))
353            .filter(|(_, f)| f.contains(x, y))
354            .min_by(|a, b| a.1.area().total_cmp(&b.1.area()))
355            .map(|(e, _)| e)
356    }
357
358    /// A short description of the screen for error reports: app names and the
359    /// first few labels.
360    pub fn summary(&self, max_labels: usize) -> serde_json::Value {
361        let labels: Vec<&str> = self
362            .elements
363            .iter()
364            .filter(|e| e.role != "application")
365            .filter_map(|e| e.label.as_deref())
366            .take(max_labels)
367            .collect();
368        serde_json::json!({
369            "apps": self.apps.iter().filter_map(|a| a.name.clone()).collect::<Vec<_>>(),
370            "element_count": self.elements.len(),
371            "labels": labels,
372        })
373    }
374}
375
376#[cfg(test)]
377mod tests {
378    use super::*;
379
380    #[test]
381    fn parses_rect_strings() {
382        let r = Rect::parse("{{20, 59}, {44, 44}}").unwrap();
383        assert_eq!((r.x, r.y, r.w, r.h), (20.0, 59.0, 44.0, 44.0));
384        let r = Rect::parse("{{168.66666666666669, 70.666666666666657}, {92.3, 20.67}}").unwrap();
385        assert_eq!(r.x, 168.7);
386        assert_eq!(r.center().0, 168.7 + 92.3 / 2.0);
387        assert!(Rect::parse("{{nan, 0}, {1, 1}}").is_none());
388        assert!(Rect::parse("{1, 2}").is_none());
389    }
390
391    #[test]
392    fn intersection_detects_offscreen() {
393        let screen = Rect { x: 0.0, y: 0.0, w: 430.0, h: 932.0 };
394        let below = Rect { x: 20.0, y: 1000.0, w: 390.0, h: 50.0 };
395        let partly = Rect { x: 20.0, y: 900.0, w: 390.0, h: 50.0 };
396        assert!(below.intersection(&screen).is_none());
397        assert!(partly.intersection(&screen).is_some());
398    }
399}