protocol/
snapshot.rs

1//! The snapshot payload: the result of a `snapshot` request.
2//!
3//! Class names, package names and resource-id prefixes live once in the
4//! snapshot's string table and nodes refer to them by index; enums are
5//! integers and node states a bitmask (design D4). Snapshots, windows, nodes
6//! and scroll semantics are maps keyed by field index; bounds, scroll ranges
7//! and scroll positions are positional arrays.
8
9use minicbor::decode::{Decoder, Error};
10use minicbor::{Decode, Encode};
11
12use crate::cbor::{self, Elements};
13
14/// How deeply nodes may nest below a window root; agents cap their walk at
15/// this depth and decoders reject deeper trees. The decoder keeps its own
16/// stack of open nodes on the heap, so a deep tree never exhausts the thread
17/// stack.
18pub const MAX_NODE_DEPTH: usize = 256;
19
20/// One tree read across the interactive windows.
21#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
22#[cbor(map)]
23pub struct Snapshot {
24    /// The string table that [`Node::class`], [`Node::res_pkg`] and
25    /// [`Window::package`] index into.
26    #[cbor(n(0), decode_with = "crate::cbor::decode_vec")]
27    pub strings: Vec<String>,
28    /// Interactive windows, top-most first.
29    #[cbor(n(1), decode_with = "crate::cbor::decode_vec")]
30    pub windows: Vec<Window>,
31    /// Nodes in this snapshot, across all windows.
32    #[n(2)]
33    pub node_count: u32,
34    /// Which bounds cut the walk short; empty when the tree is complete.
35    #[n(3)]
36    pub truncated: Truncation,
37    /// The last tree-stream event sequence number the agent had sent when it
38    /// read the tree, so a consumer knows which events the snapshot reflects.
39    #[n(4)]
40    pub seq: u64,
41}
42
43/// One window and its node tree.
44#[derive(Clone, Debug, PartialEq, Eq, Encode, Decode)]
45#[cbor(map)]
46pub struct Window {
47    /// Platform window id; events name windows by it.
48    #[n(0)]
49    pub id: u32,
50    #[n(1)]
51    pub kind: WindowType,
52    /// Z-order; higher is closer to the user.
53    #[n(2)]
54    pub layer: i32,
55    /// String-table index of the owning package name.
56    #[n(3)]
57    pub package: Option<u32>,
58    #[n(4)]
59    pub title: Option<String>,
60    #[n(5)]
61    pub bounds: Bounds,
62    #[n(6)]
63    pub flags: WindowFlags,
64    /// The root node; absent when the platform returned none.
65    #[n(7)]
66    pub root: Option<Node>,
67    /// Whether the window's toolkit sends change events for its content.
68    /// `Some(false)` marks a window whose changes are only seen by reading
69    /// it again (Jetpack Compose sends no events to a UI automation client);
70    /// `None` when the agent cannot tell.
71    #[n(8)]
72    pub emits_events: Option<bool>,
73    /// String-table index of the system process that owns the window, such
74    /// as `com.android.systemui` for the status bar or the permission
75    /// controller for a permission prompt; absent when the application under
76    /// automation owns it.
77    #[n(9)]
78    pub owner: Option<u32>,
79    /// The window is a modal, such as a dialog, that keeps the user from the
80    /// windows below it until it is dismissed; the wire omits it when false.
81    #[cbor(n(10), skip_if = "is_false")]
82    pub modal: bool,
83}
84
85/// One accessibility node.
86///
87/// Decoded without recursion (see [`MAX_NODE_DEPTH`]): the derived decoder
88/// took about 20 KiB of stack per level in a debug build.
89#[derive(Clone, Debug, PartialEq, Eq, Encode)]
90#[cbor(map)]
91pub struct Node {
92    /// Unique within the snapshot.
93    #[n(0)]
94    pub id: u32,
95    /// String-table index of the class name.
96    #[n(1)]
97    pub class: u32,
98    /// String-table index of the resource-id prefix, such as `pkg:id/`.
99    #[n(2)]
100    pub res_pkg: Option<u32>,
101    /// The resource id after its prefix.
102    #[n(3)]
103    pub res_id: Option<String>,
104    #[n(4)]
105    pub text: Option<String>,
106    /// Content description.
107    #[n(5)]
108    pub desc: Option<String>,
109    #[n(6)]
110    pub bounds: Bounds,
111    #[n(7)]
112    pub flags: NodeFlags,
113    /// Child nodes in platform order; the wire omits an empty list.
114    #[cbor(n(8), skip_if = "Vec::is_empty")]
115    pub children: Vec<Node>,
116    /// How the node scrolls; present on nodes that scroll.
117    #[n(9)]
118    pub scroll: Option<Scroll>,
119}
120
121/// Scroll semantics of a node that scrolls: the axis, whether it can scroll
122/// back toward the start of its content and forward toward its end from
123/// where it is, and, where the platform reports them, the range its content
124/// spans and the part of that range inside its viewport, both in the
125/// range's unit.
126#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encode, Decode)]
127#[cbor(map)]
128pub struct Scroll {
129    #[n(0)]
130    pub axis: ScrollAxis,
131    #[n(1)]
132    pub flags: ScrollFlags,
133    /// The range the content spans, when the platform reports one.
134    #[n(2)]
135    pub range: Option<ScrollRange>,
136    /// The part of `range` inside the viewport, when known; never present
137    /// without a range.
138    #[n(3)]
139    pub position: Option<ScrollPosition>,
140}
141
142/// The range a scrolling node's content spans, on the wire `[unit, min, max]`,
143/// both ends inclusive: `[items, 0, 199]` for a list of 200 items.
144#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encode, Decode)]
145#[cbor(array)]
146pub struct ScrollRange {
147    #[n(0)]
148    pub unit: ScrollUnit,
149    #[n(1)]
150    pub min: i32,
151    #[n(2)]
152    pub max: i32,
153}
154
155/// The part of a [`ScrollRange`] inside the viewport, on the wire
156/// `[first, last]`, both inclusive: `[0, 15]` while the first 16 items of a
157/// list are in view.
158#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Encode, Decode)]
159#[cbor(array)]
160pub struct ScrollPosition {
161    #[n(0)]
162    pub first: i32,
163    #[n(1)]
164    pub last: i32,
165}
166
167/// A node whose map is being decoded: the fields read so far and, while
168/// its child list is being read, the cursor into that list.
169struct OpenNode {
170    at: usize,
171    entries: Elements,
172    children: Option<Elements>,
173    id: Option<u32>,
174    class: Option<u32>,
175    res_pkg: Option<u32>,
176    res_id: Option<String>,
177    text: Option<String>,
178    desc: Option<String>,
179    bounds: Option<Bounds>,
180    flags: Option<NodeFlags>,
181    child_nodes: Vec<Node>,
182    scroll: Option<Scroll>,
183}
184
185impl OpenNode {
186    fn begin(d: &mut Decoder<'_>) -> Result<OpenNode, Error> {
187        Ok(OpenNode {
188            at: d.position(),
189            entries: Elements::begin_map(d)?,
190            children: None,
191            id: None,
192            class: None,
193            res_pkg: None,
194            res_id: None,
195            text: None,
196            desc: None,
197            bounds: None,
198            flags: None,
199            child_nodes: Vec::new(),
200            scroll: None,
201        })
202    }
203
204    /// Reads the value of `key`, except the child list, whose elements the
205    /// caller reads one node at a time.
206    fn read<'b, C>(&mut self, key: i64, d: &mut Decoder<'b>, ctx: &mut C) -> Result<(), Error> {
207        match key {
208            0 => self.id = Some(d.u32()?),
209            1 => self.class = Some(d.u32()?),
210            2 => self.res_pkg = Option::<u32>::decode(d, ctx)?,
211            3 => self.res_id = Option::<String>::decode(d, ctx)?,
212            4 => self.text = Option::<String>::decode(d, ctx)?,
213            5 => self.desc = Option::<String>::decode(d, ctx)?,
214            6 => self.bounds = Some(Bounds::decode(d, ctx)?),
215            7 => self.flags = Some(NodeFlags::decode(d, ctx)?),
216            8 => {
217                let children = Elements::begin(d)?;
218                self.child_nodes = Vec::with_capacity(cbor::reserve(children.len(), d));
219                self.children = Some(children);
220            }
221            9 => self.scroll = Option::<Scroll>::decode(d, ctx)?,
222            _ => d.skip()?,
223        }
224        Ok(())
225    }
226
227    fn finish(self) -> Result<Node, Error> {
228        let at = self.at;
229        Ok(Node {
230            id: self.id.ok_or_else(|| cbor::missing(0, "id").at(at))?,
231            class: self.class.ok_or_else(|| cbor::missing(1, "class").at(at))?,
232            res_pkg: self.res_pkg,
233            res_id: self.res_id,
234            text: self.text,
235            desc: self.desc,
236            bounds: self
237                .bounds
238                .ok_or_else(|| cbor::missing(6, "bounds").at(at))?,
239            flags: self.flags.ok_or_else(|| cbor::missing(7, "flags").at(at))?,
240            children: self.child_nodes,
241            scroll: self.scroll,
242        })
243    }
244}
245
246impl<'b, C> Decode<'b, C> for Node {
247    fn decode(d: &mut Decoder<'b>, ctx: &mut C) -> Result<Node, Error> {
248        let mut open = vec![OpenNode::begin(d)?];
249        loop {
250            let depth = open.len();
251            let top = open
252                .last_mut()
253                .expect("a node is open until the root finishes");
254            let child_next = match &mut top.children {
255                Some(children) => {
256                    let more = children.next(d)?;
257                    if !more {
258                        top.children = None;
259                    }
260                    more
261                }
262                None => false,
263            };
264            if child_next {
265                if depth > MAX_NODE_DEPTH {
266                    return Err(Error::message(format!(
267                        "node tree deeper than {MAX_NODE_DEPTH} levels"
268                    ))
269                    .at(d.position()));
270                }
271                open.push(OpenNode::begin(d)?);
272            } else if top.entries.next(d)? {
273                let key = d.i64()?;
274                top.read(key, d, ctx)?;
275            } else {
276                let done = open.pop().expect("checked above").finish()?;
277                match open.last_mut() {
278                    Some(parent) => parent.child_nodes.push(done),
279                    None => return Ok(done),
280                }
281            }
282        }
283    }
284}
285
286fn is_false(value: &bool) -> bool {
287    !*value
288}
289
290/// A screen rectangle in pixels, on the wire `[left, top, right, bottom]`.
291#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Encode, Decode)]
292#[cbor(array)]
293pub struct Bounds {
294    #[n(0)]
295    pub left: i32,
296    #[n(1)]
297    pub top: i32,
298    #[n(2)]
299    pub right: i32,
300    #[n(3)]
301    pub bottom: i32,
302}
303
304impl Bounds {
305    pub const fn new(left: i32, top: i32, right: i32, bottom: i32) -> Bounds {
306        Bounds {
307            left,
308            top,
309            right,
310            bottom,
311        }
312    }
313}
314
315wire_enum! {
316    /// The kind of a window, with Android's `AccessibilityWindowInfo.TYPE_*` values.
317    pub struct WindowType(u8);
318    /// Any window this version does not classify.
319    UNKNOWN = 0, "unknown";
320    APPLICATION = 1, "application";
321    INPUT_METHOD = 2, "input_method";
322    SYSTEM = 3, "system";
323    ACCESSIBILITY_OVERLAY = 4, "accessibility_overlay";
324    SPLIT_SCREEN_DIVIDER = 5, "split_screen_divider";
325    MAGNIFICATION_OVERLAY = 6, "magnification_overlay";
326}
327
328wire_flags! {
329    /// Node states.
330    pub struct NodeFlags;
331    CLICKABLE = 0, "clickable";
332    LONG_CLICKABLE = 1, "long_clickable";
333    FOCUSABLE = 2, "focusable";
334    FOCUSED = 3, "focused";
335    ENABLED = 4, "enabled";
336    CHECKABLE = 5, "checkable";
337    CHECKED = 6, "checked";
338    SCROLLABLE = 7, "scrollable";
339    SELECTED = 8, "selected";
340    EDITABLE = 9, "editable";
341    PASSWORD = 10, "password";
342    /// Visible to the user, as the platform reports it.
343    VISIBLE = 11, "visible";
344    ACCESSIBILITY_FOCUSED = 12, "accessibility_focused";
345    /// A view the platform marks modal, as iOS does with
346    /// `accessibilityViewIsModal`: assistive technology leaves out its
347    /// siblings, and what they hold, while it is on screen.
348    MODAL = 13, "modal";
349    /// Drawn after every sibling its bounds overlap, as Android reports the
350    /// order a parent draws its children in (`getDrawingOrder`): where they
351    /// overlap, it is what shows.
352    ON_TOP = 14, "on_top";
353}
354
355wire_flags! {
356    /// Window states.
357    pub struct WindowFlags;
358    ACTIVE = 0, "active";
359    FOCUSED = 1, "focused";
360    ACCESSIBILITY_FOCUSED = 2, "accessibility_focused";
361    PICTURE_IN_PICTURE = 3, "picture_in_picture";
362    /// Drawn away from where its layout puts it, as the platform draws a
363    /// window while it animates the window in or out: partly off the
364    /// display, or at another size than its layout's, which agents tell on
365    /// Android 14 and later. Its bounds and its nodes' are then those of a
366    /// frame of that animation.
367    TRANSFORMED = 4, "transformed";
368}
369
370wire_enum! {
371    /// The direction a node scrolls in.
372    pub struct ScrollAxis(u8);
373    /// The platform names no single direction.
374    UNKNOWN = 0, "unknown";
375    VERTICAL = 1, "vertical";
376    HORIZONTAL = 2, "horizontal";
377}
378
379wire_enum! {
380    /// What a [`ScrollRange`] counts.
381    pub struct ScrollUnit(u8);
382    /// A collection's items along the axis (Android's collection rows or
383    /// columns), numbered from 0.
384    ITEMS = 0, "items";
385    /// Percent of the scrollable range, 0 to 100, for a position the platform
386    /// reports as a value between bounds (Android's `RangeInfo`).
387    PERCENT = 1, "percent";
388}
389
390wire_flags! {
391    /// Which ways a node can scroll from where it is.
392    pub struct ScrollFlags;
393    /// Back, toward the start: up, or left in a left-to-right layout.
394    BACK = 0, "back";
395    /// Forward, toward the end: down, or right in a left-to-right layout.
396    FORWARD = 1, "forward";
397}
398
399wire_flags! {
400    /// Which request bounds cut a snapshot short.
401    pub struct Truncation;
402    /// `max_depth` was reached.
403    DEPTH = 0, "depth";
404    /// `max_nodes` was reached.
405    NODES = 1, "nodes";
406    /// A window could not be read.
407    WINDOWS = 2, "windows";
408}
protocol/snapshot.rs

protocol/
snapshot.rs
