smix_driver/landing.rs
1//! Where a selector act landed, judged from the tree it was aimed from.
2
3use smix_error::{ExpectationFailure, FailureCode, FailureInit};
4use smix_selector::Selector;
5
6use crate::{ActOutcome, ActVerdict, HitElement, tap_mismatch_is_fatal};
7
8/// Tolerance, in points, for comparing frames.
9///
10/// A frame makes a round trip — the host normalises the centre against
11/// the app frame, the runner multiplies it back — so exact equality
12/// would fail on arithmetic rather than on aim.
13mod miss_cause;
14
15const FRAME_TOLERANCE_PT: f64 = 1.0;
16
17/// What a chain of hit elements leaves out.
18#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19pub enum ChainCoverage {
20 /// Named elements only (the iOS runner): an unnamed target's absence
21 /// proves nothing.
22 NamedOnly,
23 /// Every element under the point (the Android runner): absence is a miss.
24 Every,
25}
26
27/// Where a touch aimed at a selector goes: the normalised point, the
28/// element it is aimed at, and the tree that aim was read from.
29pub type Aimed = (f64, f64, Option<HitElement>, smix_runner_wire::TreeReader);
30
31/// The tree a landing is judged from, given the tree the aim was read
32/// from: the same one, always, and decided here and nowhere else.
33///
34/// The aim and the verdict are two readings of one screen, and two
35/// readers can disagree about it. On a Compose screen coming in, the
36/// accessibility projection lags the semantics tree by 150-300 ms
37/// (measured on the fixture): a touch aimed from the semantics tree and
38/// judged from the projection in that gap focused its field and was
39/// reported `TAP_MISSED`.
40#[must_use]
41pub fn verdict_reader(aimed_from: smix_runner_client::TreeSource) -> smix_runner_wire::TreeReader {
42 match aimed_from {
43 smix_runner_client::TreeSource::Accessibility => {
44 smix_runner_wire::TreeReader::Accessibility
45 }
46 smix_runner_client::TreeSource::Semantics => smix_runner_wire::TreeReader::Semantics,
47 }
48}
49
50/// Where the touch went, for a failure message: the normalised point, the
51/// pixel the runner turned it into when it said, and the tree it was
52/// judged from.
53fn describe_touch(
54 nx: f64,
55 ny: f64,
56 landed: &smix_runner_wire::TapAtCoordResult,
57 reader: smix_runner_wire::TreeReader,
58) -> String {
59 let pixel = match (landed.x, landed.y) {
60 (Some(x), Some(y)) => format!(", pixel ({x},{y})"),
61 _ => String::new(),
62 };
63 format!(
64 "touched at ({nx:.4},{ny:.4}){pixel}, judged from the {} tree",
65 reader.name()
66 )
67}
68
69/// The verdict on a selector act, from what the runner said it touched.
70///
71/// One judgement for every platform and every act aimed by a selector —
72/// tap, double tap, long press. The iOS tap used to hold this inline and
73/// the Android acts had none, so an Android tap was never judged and a
74/// consumer's dialog confirm, pressed below the dialog, was reported as
75/// `tapped` with exit 0.
76///
77/// `aimed` is `None` when the selector resolved to a point rather than
78/// a node (text found by OCR, for one): there is no element to compare,
79/// and the outcome says so rather than pretending.
80///
81/// A runner that reports NOTHING under the point fails the step. Until
82/// now that was "could not be judged" and counted as a pass, on the
83/// reasoning that failing it would break everyone driving an older
84/// runner — which is how the Android runner, which reported nothing at
85/// all, turned every one of its misses into a pass.
86///
87/// # Errors
88///
89/// `TapMissed` when the touch was delivered to something other than the
90/// element aimed at (unless `SMIX_TAP_HIT_MISMATCH=warn`), and
91/// `DriverError` when the runner reported nothing about where it landed.
92pub fn landing_outcome(
93 selector: &Selector,
94 aim: Aimed,
95 landed: &smix_runner_wire::TapAtCoordResult,
96) -> Result<ActOutcome, ExpectationFailure> {
97 let (nx, ny, aimed, reader) = aim;
98 let chain: Vec<HitElement> = landed
99 .chain
100 .iter()
101 .map(|e| HitElement {
102 identifier: e.identifier.clone(),
103 label: e.label.clone(),
104 frame: (e.frame.x, e.frame.y, e.frame.w, e.frame.h),
105 })
106 .collect();
107 let Some(aimed) = aimed else {
108 return Ok(ActOutcome {
109 target: None,
110 observed: chain,
111 verdict: ActVerdict::Unconfirmable(
112 "the selector resolved to a coordinate but not to a node, so \
113 there is nothing to compare the tapped point against"
114 .into(),
115 ),
116 });
117 };
118 // The runner read the tree it was asked to, or the verdict is on
119 // something else. A runner with one reader (iOS) says nothing and
120 // answers from the accessibility tree, which is only that one's answer.
121 let judged_from = landed
122 .reader
123 .unwrap_or(smix_runner_wire::TreeReader::Accessibility);
124 if judged_from != reader || landed.reader_error.is_some() {
125 let why = landed
126 .reader_error
127 .clone()
128 .unwrap_or_else(|| format!("it read the {} tree instead", judged_from.name()));
129 return Err(ExpectationFailure::new(FailureInit {
130 code: Some(FailureCode::DriverError),
131 message: format!(
132 "the touch aimed at {} went in, and where it landed cannot be told: it was \
133 aimed from the {} tree and the runner could not judge it from that one ({why})",
134 describe_hit(&aimed),
135 reader.name(),
136 ),
137 selector: Some(selector.clone()),
138 ..Default::default()
139 }));
140 }
141 if chain.is_empty() && !landed.complete {
142 return Err(ExpectationFailure::new(FailureInit {
143 code: Some(FailureCode::DriverError),
144 message: format!(
145 "the touch aimed at {} went in, and the runner reported \
146 nothing about what it was delivered to — so whether it \
147 landed cannot be told",
148 describe_hit(&aimed)
149 ),
150 selector: Some(selector.clone()),
151 hint: Some(
152 "a runner older than the field that carries it answers this \
153 way; `smix runner up --force` rebuilds the runner from this \
154 smix"
155 .into(),
156 ),
157 ..Default::default()
158 }));
159 }
160 let coverage = if landed.complete {
161 ChainCoverage::Every
162 } else {
163 ChainCoverage::NamedOnly
164 };
165 // A runner that lists everything under the point and lists nothing
166 // is saying where the touch went: outside every window it can read.
167 // Measured with gesture navigation and a system dialog in front —
168 // the point was below the dialog, where no readable window reaches.
169 let verdict = if chain.is_empty() {
170 ActVerdict::Missed(format!(
171 "aimed at {} and the touch was delivered outside every window \
172 the runner can read",
173 describe_hit(&aimed)
174 ))
175 } else {
176 match (
177 tap_landed_within(&aimed, &chain, coverage),
178 &landed.uncertain,
179 ) {
180 (ActVerdict::Missed(why), Some(guess)) => ActVerdict::Unconfirmable(format!(
181 "{why} — but the runner could not be sure where the touch went: {guess}"
182 )),
183 (verdict, _) => verdict,
184 }
185 };
186 if let ActVerdict::Missed(why) = &verdict {
187 if tap_mismatch_is_fatal() {
188 return Err(ExpectationFailure::new(FailureInit {
189 code: Some(FailureCode::TapMissed),
190 message: format!(
191 "tap did not land where it aimed: {why} ({})",
192 describe_touch(nx, ny, landed, reader)
193 ),
194 selector: Some(selector.clone()),
195 hint: Some(
196 "the element moved between the tree fetch and the tap, or \
197 the touch went to something over it; wait for the screen \
198 to settle first. Set SMIX_TAP_HIT_MISMATCH=warn to \
199 downgrade this to a warning while migrating a suite."
200 .into(),
201 ),
202 ..Default::default()
203 }));
204 }
205 eprintln!("smix: warning: tap did not land where it aimed: {why}");
206 }
207 Ok(ActOutcome {
208 target: Some(aimed),
209 observed: chain,
210 verdict,
211 })
212}
213
214/// Did the touch land inside the element it aimed at?
215///
216/// `chain` is every named element containing the tapped point, as the
217/// runner found them after synthesising the touch.
218///
219/// # Why containment and not identity
220///
221/// The first version of this asked whether the element at the point
222/// *was* the element aimed at. A live tree says why that is wrong. At
223/// the centre of the first row of Settings, the named elements
224/// containing the point are:
225///
226/// ```text
227/// staticText "登录以访问iCloud数据…" area 7283
228/// button id=com.apple.settings.primaryAppleAccount area 33423
229/// application id=com.apple.Preferences area 351348
230/// ```
231///
232/// A flow aiming at that button taps its centre, and the innermost
233/// element there is the button's own label. Identity would call a
234/// perfectly good tap a miss — and text nested inside a row is what
235/// every list screen looks like. Containment gets it right: the button
236/// is on the chain.
237///
238/// # What the chain leaves out
239///
240/// The iOS runner lists named elements only; the Android runner lists
241/// every element under the point. `coverage` says which, and it decides
242/// what an unnamed target's absence means: nothing on iOS (it may just
243/// be unnamed), a miss on Android (the list is whole).
244///
245/// # WHAT THIS CANNOT SEE
246///
247/// **Occlusion.** A scrim covering the aimed element contains the
248/// point too, so this passes. The snapshot the runner walks carries no
249/// z-order (`TreeRoute.swift`: snapshots are dead frames), and
250/// `isHittable` — Apple's own answer — has been rejected here twice
251/// on purpose: it reports false for an element that is reachable in
252/// the AX tree but visually covered, which is exactly the see-through
253/// tap `SmixRunnerUITests.swift` performs deliberately, and it broke a
254/// QA-overlay assertion in v1.0.27.
255///
256/// So this closes the stale-frame half of "the tap reported success and
257/// nothing happened" and not the covered-element half. The whole chain
258/// travels in the outcome regardless, so a caller can see the scrim
259/// even when the verdict passes.
260pub fn tap_landed_within(
261 aimed: &HitElement,
262 chain: &[HitElement],
263 coverage: ChainCoverage,
264) -> ActVerdict {
265 if chain.is_empty() {
266 return ActVerdict::Missed(format!(
267 "aimed at {} and the tapped point held nothing — the element \
268 moved between the tree fetch and the tap, or its frame was \
269 stale",
270 describe_hit(aimed)
271 ));
272 }
273 if chain.iter().any(|c| same_element(aimed, c)) {
274 return ActVerdict::Confirmed;
275 }
276 if aimed.identifier.is_empty() && aimed.label.is_empty() && coverage == ChainCoverage::NamedOnly
277 {
278 return ActVerdict::Unconfirmable(format!(
279 "the element aimed at carries neither an identifier nor a \
280 label, so it cannot be looked for among the {} element(s) \
281 at the tapped point",
282 chain.len()
283 ));
284 }
285 ActVerdict::Missed(format!(
286 "aimed at {} but the tapped point is inside {} instead",
287 describe_hit(aimed),
288 chain
289 .iter()
290 .map(describe_hit)
291 .collect::<Vec<_>>()
292 .join(", ")
293 ))
294}
295
296/// Are these two descriptions the same element?
297///
298/// By the strongest field both carry: identifier, then label, then
299/// geometry.
300fn same_element(a: &HitElement, b: &HitElement) -> bool {
301 if !a.identifier.is_empty() && !b.identifier.is_empty() {
302 return a.identifier == b.identifier;
303 }
304 if !a.label.is_empty() && !b.label.is_empty() {
305 return a.label == b.label;
306 }
307 if a.identifier.is_empty()
308 && a.label.is_empty()
309 && b.identifier.is_empty()
310 && b.label.is_empty()
311 {
312 let close = |x: f64, y: f64| (x - y).abs() <= FRAME_TOLERANCE_PT;
313 return close(a.frame.0, b.frame.0)
314 && close(a.frame.1, b.frame.1)
315 && close(a.frame.2, b.frame.2)
316 && close(a.frame.3, b.frame.3);
317 }
318 // One is named and the other is not: they are describable in
319 // different vocabularies, which is not evidence of sameness.
320 false
321}
322
323pub(crate) fn describe_hit(e: &HitElement) -> String {
324 if !e.identifier.is_empty() {
325 format!("id={}", e.identifier)
326 } else if !e.label.is_empty() {
327 format!("label={:?}", e.label)
328 } else {
329 format!(
330 "an unnamed element at ({:.0},{:.0} {:.0}x{:.0})",
331 e.frame.0, e.frame.1, e.frame.2, e.frame.3
332 )
333 }
334}