android/session/
tree.rs

1//! Tree reads: the agent's `snapshot` as the mirror's `TreeRead`. The agent
2//! carries each window's metadata (whether its toolkit sends change events,
3//! the system process that owns it, whether it is modal) and each scrolling
4//! node's semantics in the snapshot itself, so the payload passes through as
5//! the agent sent it and the mirror reads them there. Bounds probes go to
6//! the agent's `bounds` when it advertises them (task 10.11), and a read for
7//! nodes the probes found moved asks the agent to evict their windows.
8
9use std::sync::atomic::Ordering;
10use std::sync::{Arc, PoisonError};
11use std::time::{Duration, Instant};
12
13use mirror::backend::{
14    BackendError, BackendInfo, NodeHandle, ProbedNode, ReadReason, ReadRequest, TreeRead,
15    TreeSource,
16};
17use protocol::event::Stream;
18use protocol::message::{BoundsParams, Call, Reply, SnapshotParams};
19use protocol::snapshot::{Node, Snapshot};
20use protocol::{Capabilities, code};
21
22use super::{INPUT_QUIET, Inner, ReadOptions, ReadStats};
23
24/// The backend name the mirror reports for reads from the agent.
25pub const BACKEND: &str = "android-agent";
26
27/// Deadline of a bounds probe: a few round trips to the app's UI thread,
28/// which a busy thread delays; past it the settle wait goes on and probes
29/// again.
30pub const PROBE_DEADLINE: Duration = Duration::from_millis(500);
31
32/// Physical pixels per device point from `wm density` output: the override
33/// density when one is set, else the physical one, over 160.
34pub fn scale_from_density(wm_density: &str) -> Option<f64> {
35    let density = |label: &str| {
36        wm_density
37            .lines()
38            .find_map(|line| line.trim().strip_prefix(label))
39            .and_then(|value| value.trim().parse::<f64>().ok())
40            .filter(|dpi| *dpi > 0.0)
41    };
42    density("Override density:")
43        .or_else(|| density("Physical density:"))
44        .map(|dpi| dpi / 160.0)
45}
46
47/// Whether a window of `snapshot` sends no change events by the agent's
48/// account, as a window of Jetpack Compose content does: its changes are
49/// seen only by reading it again.
50pub fn has_event_less_window(snapshot: &Snapshot) -> bool {
51    snapshot
52        .windows
53        .iter()
54        .any(|window| window.emits_events == Some(false))
55}
56
57/// The parameters of a read for `reason`: reads the target's own changes
58/// trigger do not wait for idle, reads after an attach, an action or a
59/// resync take the settle bound.
60pub fn snapshot_params(reason: ReadReason, options: &ReadOptions) -> SnapshotParams {
61    let idle_ms = match reason {
62        ReadReason::Attach | ReadReason::Action | ReadReason::Resync => options.settle_idle_ms,
63        ReadReason::Events | ReadReason::Poll | ReadReason::Frames | ReadReason::Moved => {
64            Some(options.idle_ms)
65        }
66    };
67    SnapshotParams {
68        max_depth: options.max_depth,
69        max_nodes: options.max_nodes,
70        idle_ms,
71        ..SnapshotParams::default()
72    }
73}
74
75/// Nodes a snapshot carries.
76pub(crate) fn serialized_nodes(snapshot: &Snapshot) -> u32 {
77    fn count(node: &Node) -> u32 {
78        1 + node.children.iter().map(count).sum::<u32>()
79    }
80    snapshot
81        .windows
82        .iter()
83        .filter_map(|w| w.root.as_ref())
84        .map(count)
85        .sum()
86}
87
88/// The session's tree source.
89pub(crate) struct AgentTree {
90    pub(crate) inner: Arc<Inner>,
91}
92
93impl TreeSource for AgentTree {
94    fn read(&mut self, request: &ReadRequest) -> Result<TreeRead, BackendError> {
95        let inner = &*self.inner;
96        let started = Instant::now();
97        let deadline = inner.options.read.deadline;
98        // A resync the host session could not complete leaves the stream's
99        // events dropped until another one succeeds.
100        let failed: Vec<Stream> = inner
101            .failed_resyncs
102            .lock()
103            .unwrap_or_else(PoisonError::into_inner)
104            .drain(..)
105            .collect();
106        for stream in failed {
107            let _ = inner.connection.resync(stream);
108        }
109        let mut cleared_cache = false;
110        let mut params = snapshot_params(request.reason, &inner.options.read);
111        if request.reason == ReadReason::Moved {
112            // Bounds probes found these windows' nodes moved without a
113            // change event: the agent's cache still holds their old bounds.
114            params.evict = request.windows.clone();
115            cleared_cache = true;
116        }
117        let foreign = inner
118            .foreign_input
119            .lock()
120            .unwrap_or_else(PoisonError::into_inner)
121            .take();
122        if let Some(input) = foreign {
123            // The agent's idle wait knows nothing of this input, and its cache
124            // of windows without events is stale after it: the read asks the
125            // agent to evict those windows alone, leaving the tree stream and
126            // the other windows' cache, which their events keep current.
127            let quiet = input + INPUT_QUIET;
128            if let Some(wait) = quiet.checked_duration_since(Instant::now()) {
129                std::thread::sleep(wait);
130            }
131            if inner.event_less.load(Ordering::SeqCst) {
132                params.evict_event_less = Some(true);
133                cleared_cache = true;
134            }
135        }
136        let Reply::Snapshot(snapshot) = inner.connection.call(Call::Snapshot(params), deadline)?
137        else {
138            return Err(BackendError::new(
139                code::PROTOCOL_ERROR,
140                "the agent answered a snapshot request with another result",
141            ));
142        };
143        inner
144            .event_less
145            .store(has_event_less_window(&snapshot), Ordering::SeqCst);
146        inner.reads.fetch_add(1, Ordering::SeqCst);
147        *inner
148            .last_read
149            .lock()
150            .unwrap_or_else(PoisonError::into_inner) = Some(ReadStats {
151            node_count: snapshot.node_count,
152            serialized: serialized_nodes(&snapshot),
153            windows: u32::try_from(snapshot.windows.len()).unwrap_or(u32::MAX),
154            took: started.elapsed(),
155            reason: request.reason,
156            cleared_cache,
157        });
158        Ok(TreeRead {
159            snapshot,
160            backend: BackendInfo::new(BACKEND, false),
161            scale: inner.scale,
162        })
163    }
164
165    fn can_probe(&self) -> bool {
166        self.inner
167            .connection
168            .negotiated()
169            .is_some_and(|n| n.capabilities.contains(Capabilities::BOUNDS))
170    }
171
172    fn probe(&mut self, nodes: &[NodeHandle]) -> Result<Vec<ProbedNode>, BackendError> {
173        let params = BoundsParams {
174            nodes: nodes.iter().map(|handle| handle.node).collect(),
175        };
176        let Reply::Bounds(probed) = self
177            .inner
178            .connection
179            .call(Call::Bounds(params), PROBE_DEADLINE)?
180        else {
181            return Err(BackendError::new(
182                code::PROTOCOL_ERROR,
183                "the agent answered a bounds probe with another result",
184            ));
185        };
186        // Node ids are unique across the windows of one snapshot.
187        Ok(probed
188            .nodes
189            .into_iter()
190            .filter_map(|found| {
191                let handle = nodes.iter().find(|h| h.node == found.node)?;
192                Some(ProbedNode {
193                    node: *handle,
194                    bounds: found.bounds,
195                })
196            })
197            .collect())
198    }
199}
200
201#[cfg(test)]
202pub(crate) mod tests {
203    use super::*;
204    use protocol::snapshot::{Bounds, NodeFlags, Truncation, Window, WindowFlags, WindowType};
205
206    pub(crate) fn node(id: u32, class: u32, children: Vec<Node>) -> Node {
207        Node {
208            id,
209            class,
210            res_pkg: None,
211            res_id: None,
212            text: None,
213            desc: None,
214            bounds: Bounds::new(0, 0, 100, 100),
215            flags: NodeFlags::ENABLED | NodeFlags::VISIBLE,
216            children,
217            scroll: None,
218        }
219    }
220
221    pub(crate) fn window(
222        id: u32,
223        kind: WindowType,
224        layer: i32,
225        package: u32,
226        bounds: Bounds,
227        root: Node,
228    ) -> Window {
229        Window {
230            id,
231            kind,
232            layer,
233            package: Some(package),
234            title: None,
235            bounds,
236            flags: WindowFlags::EMPTY,
237            root: Some(root),
238            emits_events: None,
239            owner: None,
240            modal: false,
241        }
242    }
243
244    /// `window` with the metadata the agent reports: whether it sends change
245    /// events, and its system owner as a string-table index.
246    pub(crate) fn reported(window: Window, emits_events: bool, owner: Option<u32>) -> Window {
247        Window {
248            emits_events: Some(emits_events),
249            owner,
250            ..window
251        }
252    }
253
254    /// Strings: 0 FrameLayout, 1 ComposeView, 2 com.android.systemui,
255    /// 3 the fixture, 4 Button, 5 the keyboard's package.
256    pub(crate) fn strings() -> Vec<String> {
257        [
258            "android.widget.FrameLayout",
259            "androidx.compose.ui.platform.ComposeView",
260            "com.android.systemui",
261            "dev.mobileagent.fixture.compose",
262            "android.widget.Button",
263            "com.google.android.inputmethod.latin",
264        ]
265        .map(String::from)
266        .to_vec()
267    }
268
269    /// A keyboard, a status bar and a Compose app window, top-most first,
270    /// with the metadata the agent reports for them.
271    pub(crate) fn compose_screen(cells: u32) -> Snapshot {
272        let full = Bounds::new(0, 0, 1080, 2400);
273        let buttons = (0..cells).map(|i| node(10 + i, 4, Vec::new())).collect();
274        Snapshot {
275            strings: strings(),
276            windows: vec![
277                reported(
278                    window(
279                        3,
280                        WindowType::INPUT_METHOD,
281                        3,
282                        5,
283                        Bounds::new(0, 1500, 1080, 2400),
284                        node(1, 0, Vec::new()),
285                    ),
286                    true,
287                    Some(5),
288                ),
289                reported(
290                    window(
291                        2,
292                        WindowType::SYSTEM,
293                        2,
294                        2,
295                        Bounds::new(0, 0, 1080, 132),
296                        node(2, 0, Vec::new()),
297                    ),
298                    true,
299                    Some(2),
300                ),
301                reported(
302                    window(
303                        1,
304                        WindowType::APPLICATION,
305                        1,
306                        3,
307                        full,
308                        node(3, 0, vec![node(4, 1, buttons)]),
309                    ),
310                    false,
311                    None,
312                ),
313            ],
314            node_count: 4 + cells,
315            truncated: Truncation::EMPTY,
316            seq: 0,
317        }
318    }
319
320    #[test]
321    fn the_scale_is_the_density_over_160_with_the_override_first() {
322        assert_eq!(scale_from_density("Physical density: 420\n"), Some(2.625));
323        assert_eq!(
324            scale_from_density("Physical density: 420\nOverride density: 480\n"),
325            Some(3.0)
326        );
327        assert_eq!(scale_from_density("Physical density: 160"), Some(1.0));
328        assert_eq!(scale_from_density("Error: no display"), None);
329    }
330
331    #[test]
332    fn event_less_windows_come_from_the_agents_metadata() {
333        let mut snapshot = compose_screen(3);
334        assert!(has_event_less_window(&snapshot));
335        snapshot.windows[2].emits_events = Some(true);
336        assert!(!has_event_less_window(&snapshot));
337        // A window the agent could not judge, such as one without a root.
338        snapshot.windows[2].emits_events = None;
339        assert!(!has_event_less_window(&snapshot));
340    }
341
342    #[test]
343    fn reads_after_an_attach_action_or_resync_wait_for_idle_and_others_do_not() {
344        let options = ReadOptions {
345            max_nodes: Some(1000),
346            max_depth: Some(40),
347            idle_ms: 0,
348            settle_idle_ms: Some(300),
349            ..ReadOptions::default()
350        };
351        for reason in [ReadReason::Attach, ReadReason::Action, ReadReason::Resync] {
352            let params = snapshot_params(reason, &options);
353            assert_eq!(params.idle_ms, Some(300), "{reason:?}");
354            assert_eq!(params.max_nodes, Some(1000));
355            assert_eq!(params.max_depth, Some(40));
356        }
357        for reason in [
358            ReadReason::Events,
359            ReadReason::Poll,
360            ReadReason::Frames,
361            ReadReason::Moved,
362        ] {
363            assert_eq!(
364                snapshot_params(reason, &options).idle_ms,
365                Some(0),
366                "{reason:?}"
367            );
368        }
369        let defaults = snapshot_params(ReadReason::Attach, &ReadOptions::default());
370        assert_eq!(defaults, SnapshotParams::default(), "the agent's defaults");
371    }
372
373    #[test]
374    fn serialized_nodes_count_every_window() {
375        assert_eq!(serialized_nodes(&compose_screen(5)), 9);
376    }
377}
android/session/tree.rs

android/session/
tree.rs
