Skip to main content

iphone_cli/ax/
tree.rs

1//! Turn a raw dump into the element list `iph read` prints.
2
3use std::collections::HashSet;
4
5use serde::{Deserialize, Serialize};
6
7use super::{
8    archive::Ka,
9    attrs::{self, traits},
10    dump::{Dump, ObjKey, RawElement},
11};
12
13#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
14pub struct Rect {
15    pub x: f64,
16    pub y: f64,
17    pub w: f64,
18    pub h: f64,
19}
20
21impl Rect {
22    pub fn center(&self) -> (f64, f64) {
23        (self.x + self.w / 2.0, self.y + self.h / 2.0)
24    }
25
26    pub fn contains(&self, x: f64, y: f64) -> bool {
27        x >= self.x && x <= self.x + self.w && y >= self.y && y <= self.y + self.h
28    }
29
30    pub fn area(&self) -> f64 {
31        self.w.max(0.0) * self.h.max(0.0)
32    }
33
34    pub fn intersection(&self, other: &Rect) -> Option<Rect> {
35        let x1 = self.x.max(other.x);
36        let y1 = self.y.max(other.y);
37        let x2 = (self.x + self.w).min(other.x + other.w);
38        let y2 = (self.y + self.h).min(other.y + other.h);
39        (x2 > x1 && y2 > y1).then_some(Rect {
40            x: x1,
41            y: y1,
42            w: x2 - x1,
43            h: y2 - y1,
44        })
45    }
46
47    /// Parse `{{x, y}, {w, h}}`.
48    pub fn parse(s: &str) -> Option<Rect> {
49        let nums: Vec<f64> = s
50            .split(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace())
51            .filter(|p| !p.is_empty())
52            .map(|p| p.parse::<f64>())
53            .collect::<Result<_, _>>()
54            .ok()?;
55        if nums.len() != 4 || nums.iter().any(|n| !n.is_finite()) {
56            return None;
57        }
58        Some(Rect {
59            x: round1(nums[0]),
60            y: round1(nums[1]),
61            w: round1(nums[2]),
62            h: round1(nums[3]),
63        })
64    }
65}
66
67fn round1(v: f64) -> f64 {
68    (v * 10.0).round() / 10.0
69}
70
71#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
72pub struct Element {
73    /// Short handle for this element, stable while the element lives.
74    pub id: String,
75    pub role: String,
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub label: Option<String>,
78    #[serde(skip_serializing_if = "Option::is_none")]
79    pub value: Option<String>,
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub identifier: Option<String>,
82    #[serde(default, skip_serializing_if = "Option::is_none")]
83    pub hint: Option<String>,
84    #[serde(skip_serializing_if = "Option::is_none")]
85    pub frame: Option<Rect>,
86    #[serde(default)]
87    pub traits: Vec<String>,
88    #[serde(default = "yes")]
89    pub enabled: bool,
90    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
91    pub selected: bool,
92    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
93    pub focused: bool,
94    /// `id` of the parent element, absent for an application.
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub parent: Option<String>,
97    #[serde(default)]
98    pub depth: u32,
99    #[serde(default)]
100    pub pid: i64,
101    /// False when the element lies outside the screen.
102    #[serde(default = "yes")]
103    pub on_screen: bool,
104    /// Every numbered attribute, only with `--raw`.
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub raw: Option<serde_json::Value>,
107    /// The phone's name for the element, to ask it about one attribute.
108    #[serde(skip)]
109    pub key: Option<ObjKey>,
110}
111
112fn yes() -> bool {
113    true
114}
115
116#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
117pub struct AppInfo {
118    pub pid: i64,
119    #[serde(skip_serializing_if = "Option::is_none")]
120    pub name: Option<String>,
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub bundle_id: Option<String>,
123}
124
125#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
126pub struct Screen {
127    /// Logical screen size in points.
128    pub width: f64,
129    pub height: f64,
130    pub apps: Vec<AppInfo>,
131    pub elements: Vec<Element>,
132}
133
134/// Role, label, value and rounded frame of one element.
135pub type ElementSignature = (String, Option<String>, Option<String>, Option<(i64, i64, i64, i64)>);
136
137#[derive(Debug, Clone, Copy, Default)]
138pub struct BuildOptions {
139    /// Keep elements that lie outside the screen and walk hidden children.
140    pub include_offscreen: bool,
141    /// Attach every numbered attribute to each element.
142    pub raw: bool,
143}
144
145fn handle(key: &ObjKey) -> String {
146    // Fold the 64-bit object id and the pid into 32 bits; unique enough within
147    // one screen, and `Screen::build` de-duplicates the rare collision.
148    let mut h = key.object_id ^ (key.object_id >> 32) ^ ((key.pid as u64) << 17);
149    h = h.wrapping_mul(0x9E37_79B9_7F4A_7C15);
150    format!("{:08x}", (h >> 32) as u32)
151}
152
153/// A readable value, including an empty or whitespace-only string. Missing
154/// attributes and unsupported values are not evidence of an empty field.
155pub(crate) fn value_text(v: Option<&Ka>) -> Option<String> {
156    Some(match v? {
157        Ka::Str(s) => s.clone(),
158        other @ Ka::Obj { .. } => other.text()?,
159        Ka::Int(i) => i.to_string(),
160        Ka::UInt(u) => u.to_string(),
161        Ka::Real(r) => r.to_string(),
162        _ => return None,
163    })
164}
165
166pub(crate) fn text(v: Option<&Ka>) -> Option<String> {
167    let s = value_text(v)?;
168    let t = s.trim();
169    (!t.is_empty()).then(|| t.to_string())
170}
171
172fn frame_of(raw: &RawElement) -> Option<Rect> {
173    raw.attrs
174        .get(&attrs::FRAME)
175        .and_then(|v| v.get("NS.rectval"))
176        .and_then(|v| v.as_str())
177        .and_then(Rect::parse)
178}
179
180pub(crate) fn children_of(raw: &RawElement, include_offscreen: bool) -> Vec<ObjKey> {
181    let pick = |n: i64| -> Vec<ObjKey> {
182        raw.attrs
183            .get(&n)
184            .and_then(|v| v.as_array())
185            .map(|a| a.iter().filter_map(ObjKey::from_ka).collect())
186            .unwrap_or_default()
187    };
188    let (first, second) = if include_offscreen {
189        (attrs::CHILDREN, attrs::VISIBLE_CHILDREN)
190    } else {
191        (attrs::VISIBLE_CHILDREN, attrs::CHILDREN)
192    };
193    let list = pick(first);
194    if list.is_empty() { pick(second) } else { list }
195}
196
197impl Screen {
198    pub fn build(dump: &Dump, opts: BuildOptions) -> Screen {
199        let mut screen = Screen {
200            width: 0.0,
201            height: 0.0,
202            apps: Vec::new(),
203            elements: Vec::new(),
204        };
205        // The screen is as large as the largest application frame.
206        for key in &dump.applications {
207            if let Some(f) = dump.elements.get(key).and_then(frame_of) {
208                screen.width = screen.width.max(f.x + f.w);
209                screen.height = screen.height.max(f.y + f.h);
210            }
211        }
212        let bounds = Rect {
213            x: 0.0,
214            y: 0.0,
215            w: screen.width,
216            h: screen.height,
217        };
218
219        let mut seen: HashSet<ObjKey> = HashSet::new();
220        let mut used_ids: HashSet<String> = HashSet::new();
221        // Depth-first from each application, in the phone's own child order.
222        let mut stack: Vec<(ObjKey, Option<String>, u32)> = dump
223            .applications
224            .iter()
225            .rev()
226            .map(|k| (*k, None, 0))
227            .collect();
228        // Elements the phone described but no application reaches (rare).
229        let mut orphans: Vec<ObjKey> = Vec::new();
230
231        loop {
232            let Some((key, parent, depth)) = stack.pop() else {
233                if orphans.is_empty() {
234                    orphans = dump
235                        .order
236                        .iter()
237                        .filter(|k| !seen.contains(k))
238                        .copied()
239                        .collect();
240                    if orphans.is_empty() {
241                        break;
242                    }
243                    for k in orphans.iter().rev() {
244                        stack.push((*k, None, 1));
245                    }
246                    continue;
247                }
248                break;
249            };
250            if !seen.insert(key) {
251                continue;
252            }
253            let Some(raw) = dump.elements.get(&key) else {
254                continue;
255            };
256            let bits = raw
257                .attrs
258                .get(&attrs::TRAITS)
259                .and_then(|v| v.as_u64())
260                .unwrap_or(0);
261            let role_num = raw.attrs.get(&attrs::ROLE).and_then(|v| v.as_i64());
262            let frame = frame_of(raw);
263            let on_screen = match (&frame, bounds.area() > 0.0) {
264                (Some(f), true) => f.intersection(&bounds).is_some(),
265                (Some(f), false) => f.area() > 0.0,
266                (None, _) => raw.is_application,
267            };
268
269            let mut id = handle(&key);
270            while !used_ids.insert(id.clone()) {
271                id.push('x');
272            }
273            for child in children_of(raw, opts.include_offscreen).into_iter().rev() {
274                stack.push((child, Some(id.clone()), depth + 1));
275            }
276            if raw.is_application {
277                screen.apps.push(AppInfo {
278                    pid: key.pid,
279                    name: text(raw.attrs.get(&attrs::LABEL))
280                        .or_else(|| text(raw.attrs.get(&attrs::LABEL_PLAIN))),
281                    bundle_id: None,
282                });
283            }
284            if !on_screen && !opts.include_offscreen && !raw.is_application {
285                continue;
286            }
287            let label = text(raw.attrs.get(&attrs::LABEL))
288                .or_else(|| text(raw.attrs.get(&attrs::LABEL_PLAIN)));
289            let value = value_text(raw.attrs.get(&attrs::VALUE));
290            let has_text = label.is_some() || value.as_ref().is_some_and(|v| !v.trim().is_empty());
291            screen.elements.push(Element {
292                id,
293                role: attrs::role_name(role_num, bits, raw.is_application, has_text).to_string(),
294                label,
295                value,
296                identifier: text(raw.attrs.get(&attrs::IDENTIFIER)),
297                hint: text(raw.attrs.get(&attrs::HINT)),
298                frame,
299                traits: attrs::trait_names(bits)
300                    .into_iter()
301                    .map(String::from)
302                    .collect(),
303                enabled: bits & traits::NOT_ENABLED == 0,
304                selected: bits & traits::SELECTED != 0,
305                focused: bits & traits::IS_EDITING != 0,
306                parent,
307                depth,
308                pid: key.pid,
309                key: Some(key),
310                on_screen,
311                raw: opts.raw.then(|| {
312                    let mut m = serde_json::Map::new();
313                    for (n, v) in &raw.attrs {
314                        m.insert(n.to_string(), v.to_json());
315                    }
316                    for (n, v) in &raw.singles {
317                        m.insert(format!("single_{n}"), v.to_json());
318                    }
319                    serde_json::Value::Object(m)
320                }),
321            });
322        }
323        screen
324    }
325
326    /// What a comparison of two readings looks at: everything a user could
327    /// see or an agent could act on, without handles.
328    pub fn signature(&self) -> Vec<ElementSignature> {
329        self.elements
330            .iter()
331            .map(|e| {
332                (
333                    e.role.clone(),
334                    e.label.clone(),
335                    e.value.clone(),
336                    e.frame.map(|f| {
337                        (
338                            f.x.round() as i64,
339                            f.y.round() as i64,
340                            f.w.round() as i64,
341                            f.h.round() as i64,
342                        )
343                    }),
344                )
345            })
346            .collect()
347    }
348
349    pub fn by_id(&self, id: &str) -> Option<&Element> {
350        self.elements.iter().find(|e| e.id == id)
351    }
352
353    /// The smallest on-screen element whose frame contains the point.
354    pub fn element_at(&self, x: f64, y: f64) -> Option<&Element> {
355        self.elements
356            .iter()
357            .filter(|e| e.role != "application")
358            .filter_map(|e| e.frame.map(|f| (e, f)))
359            .filter(|(_, f)| f.contains(x, y))
360            .min_by(|a, b| a.1.area().total_cmp(&b.1.area()))
361            .map(|(e, _)| e)
362    }
363
364    /// A short description of the screen for error reports: app names and the
365    /// first few labels.
366    pub fn summary(&self, max_labels: usize) -> serde_json::Value {
367        let labels: Vec<&str> = self
368            .elements
369            .iter()
370            .filter(|e| e.role != "application")
371            .filter_map(|e| e.label.as_deref())
372            .take(max_labels)
373            .collect();
374        serde_json::json!({
375            "apps": self.apps.iter().filter_map(|a| a.name.clone()).collect::<Vec<_>>(),
376            "element_count": self.elements.len(),
377            "labels": labels,
378        })
379    }
380}
381
382#[cfg(test)]
383mod tests {
384    use super::*;
385
386    #[test]
387    fn field_values_keep_empty_and_whitespace_distinct_from_missing() {
388        for (raw, expected) in [
389            (Ka::Str(String::new()), Some("")),
390            (Ka::Str(" \n ".into()), Some(" \n ")),
391            (Ka::Str("  hello  ".into()), Some("  hello  ")),
392            (Ka::Int(0), Some("0")),
393            (Ka::Null, None),
394            (Ka::Bool(false), None),
395            (
396                Ka::Obj {
397                    class: "Unknown".into(),
398                    fields: vec![],
399                },
400                None,
401            ),
402        ] {
403            assert_eq!(value_text(Some(&raw)).as_deref(), expected);
404        }
405        assert_eq!(value_text(None), None);
406        // Labels keep their existing whitespace normalization.
407        assert_eq!(
408            text(Some(&Ka::Str("  hello  ".into()))).as_deref(),
409            Some("hello")
410        );
411    }
412
413    #[test]
414    fn screen_build_preserves_an_empty_field_value() {
415        use crate::ax::dump::DumpKind;
416        let key = ObjKey {
417            pid: 10,
418            object_id: 1,
419        };
420        let mut dump = Dump {
421            kind: DumpKind::Initial,
422            applications: vec![key],
423            elements: [(
424                key,
425                RawElement {
426                    is_application: true,
427                    ..Default::default()
428                },
429            )]
430            .into(),
431            order: vec![key],
432        };
433        let missing = Screen::build(&dump, BuildOptions::default());
434        dump.elements
435            .get_mut(&key)
436            .unwrap()
437            .attrs
438            .insert(attrs::VALUE, Ka::Str(String::new()));
439        let empty = Screen::build(&dump, BuildOptions::default());
440        assert_eq!(missing.elements[0].value, None);
441        assert_eq!(empty.elements[0].value.as_deref(), Some(""));
442        assert_ne!(missing.signature(), empty.signature());
443    }
444
445    #[test]
446    fn parses_rect_strings() {
447        let r = Rect::parse("{{20, 59}, {44, 44}}").unwrap();
448        assert_eq!((r.x, r.y, r.w, r.h), (20.0, 59.0, 44.0, 44.0));
449        let r = Rect::parse("{{168.66666666666669, 70.666666666666657}, {92.3, 20.67}}").unwrap();
450        assert_eq!(r.x, 168.7);
451        assert_eq!(r.center().0, 168.7 + 92.3 / 2.0);
452        assert!(Rect::parse("{{nan, 0}, {1, 1}}").is_none());
453        assert!(Rect::parse("{1, 2}").is_none());
454    }
455
456    #[test]
457    fn intersection_detects_offscreen() {
458        let screen = Rect { x: 0.0, y: 0.0, w: 430.0, h: 932.0 };
459        let below = Rect { x: 20.0, y: 1000.0, w: 390.0, h: 50.0 };
460        let partly = Rect { x: 20.0, y: 900.0, w: 390.0, h: 50.0 };
461        assert!(below.intersection(&screen).is_none());
462        assert!(partly.intersection(&screen).is_some());
463    }
464}