Skip to main content

smix_driver/
landing.rs

1//! Where a selector act landed, judged from the tree it was aimed from.
2
3use smix_error::{ExpectationFailure, FailureCode, FailureInit};
4use smix_selector::Selector;
5
6use crate::{ActOutcome, ActVerdict, HitElement, tap_mismatch_is_fatal};
7
8/// Tolerance, in points, for comparing frames.
9///
10/// A frame makes a round trip — the host normalises the centre against
11/// the app frame, the runner multiplies it back — so exact equality
12/// would fail on arithmetic rather than on aim.
13mod miss_cause;
14
15const FRAME_TOLERANCE_PT: f64 = 1.0;
16
17/// What a chain of hit elements leaves out.
18#[derive(Clone, Copy, Debug, PartialEq, Eq)]
19pub enum ChainCoverage {
20    /// Named elements only (the iOS runner): an unnamed target's absence
21    /// proves nothing.
22    NamedOnly,
23    /// Every element under the point (the Android runner): absence is a miss.
24    Every,
25}
26
27/// Where a touch aimed at a selector goes: the normalised point, the
28/// element it is aimed at, and the tree that aim was read from.
29pub type Aimed = (f64, f64, Option<HitElement>, smix_runner_wire::TreeReader);
30
31/// The tree a landing is judged from, given the tree the aim was read
32/// from: the same one, always, and decided here and nowhere else.
33///
34/// The aim and the verdict are two readings of one screen, and two
35/// readers can disagree about it. On a Compose screen coming in, the
36/// accessibility projection lags the semantics tree by 150-300 ms
37/// (measured on the fixture): a touch aimed from the semantics tree and
38/// judged from the projection in that gap focused its field and was
39/// reported `TAP_MISSED`.
40#[must_use]
41pub fn verdict_reader(aimed_from: smix_runner_client::TreeSource) -> smix_runner_wire::TreeReader {
42    match aimed_from {
43        smix_runner_client::TreeSource::Accessibility => {
44            smix_runner_wire::TreeReader::Accessibility
45        }
46        smix_runner_client::TreeSource::Semantics => smix_runner_wire::TreeReader::Semantics,
47    }
48}
49
50/// Where the touch went, for a failure message: the normalised point, the
51/// pixel the runner turned it into when it said, and the tree it was
52/// judged from.
53fn describe_touch(
54    nx: f64,
55    ny: f64,
56    landed: &smix_runner_wire::TapAtCoordResult,
57    reader: smix_runner_wire::TreeReader,
58) -> String {
59    let pixel = match (landed.x, landed.y) {
60        (Some(x), Some(y)) => format!(", pixel ({x},{y})"),
61        _ => String::new(),
62    };
63    format!(
64        "touched at ({nx:.4},{ny:.4}){pixel}, judged from the {} tree",
65        reader.name()
66    )
67}
68
69/// The verdict on a selector act, from what the runner said it touched.
70///
71/// One judgement for every platform and every act aimed by a selector —
72/// tap, double tap, long press. The iOS tap used to hold this inline and
73/// the Android acts had none, so an Android tap was never judged and a
74/// consumer's dialog confirm, pressed below the dialog, was reported as
75/// `tapped` with exit 0.
76///
77/// `aimed` is `None` when the selector resolved to a point rather than
78/// a node (text found by OCR, for one): there is no element to compare,
79/// and the outcome says so rather than pretending.
80///
81/// A runner that reports NOTHING under the point fails the step. Until
82/// now that was "could not be judged" and counted as a pass, on the
83/// reasoning that failing it would break everyone driving an older
84/// runner — which is how the Android runner, which reported nothing at
85/// all, turned every one of its misses into a pass.
86///
87/// # Errors
88///
89/// `TapMissed` when the touch was delivered to something other than the
90/// element aimed at (unless `SMIX_TAP_HIT_MISMATCH=warn`), and
91/// `DriverError` when the runner reported nothing about where it landed.
92pub fn landing_outcome(
93    selector: &Selector,
94    aim: Aimed,
95    landed: &smix_runner_wire::TapAtCoordResult,
96) -> Result<ActOutcome, ExpectationFailure> {
97    let (nx, ny, aimed, reader) = aim;
98    let chain: Vec<HitElement> = landed
99        .chain
100        .iter()
101        .map(|e| HitElement {
102            identifier: e.identifier.clone(),
103            label: e.label.clone(),
104            frame: (e.frame.x, e.frame.y, e.frame.w, e.frame.h),
105        })
106        .collect();
107    let Some(aimed) = aimed else {
108        return Ok(ActOutcome {
109            target: None,
110            observed: chain,
111            verdict: ActVerdict::Unconfirmable(
112                "the selector resolved to a coordinate but not to a node, so \
113                 there is nothing to compare the tapped point against"
114                    .into(),
115            ),
116        });
117    };
118    // The runner read the tree it was asked to, or the verdict is on
119    // something else. A runner with one reader (iOS) says nothing and
120    // answers from the accessibility tree, which is only that one's answer.
121    let judged_from = landed
122        .reader
123        .unwrap_or(smix_runner_wire::TreeReader::Accessibility);
124    if judged_from != reader || landed.reader_error.is_some() {
125        let why = landed
126            .reader_error
127            .clone()
128            .unwrap_or_else(|| format!("it read the {} tree instead", judged_from.name()));
129        return Err(ExpectationFailure::new(FailureInit {
130            code: Some(FailureCode::DriverError),
131            message: format!(
132                "the touch aimed at {} went in, and where it landed cannot be told: it was \
133                 aimed from the {} tree and the runner could not judge it from that one ({why})",
134                describe_hit(&aimed),
135                reader.name(),
136            ),
137            selector: Some(selector.clone()),
138            ..Default::default()
139        }));
140    }
141    if chain.is_empty() && !landed.complete {
142        return Err(ExpectationFailure::new(FailureInit {
143            code: Some(FailureCode::DriverError),
144            message: format!(
145                "the touch aimed at {} went in, and the runner reported \
146                 nothing about what it was delivered to — so whether it \
147                 landed cannot be told",
148                describe_hit(&aimed)
149            ),
150            selector: Some(selector.clone()),
151            hint: Some(
152                "a runner older than the field that carries it answers this \
153                 way; `smix runner up --force` rebuilds the runner from this \
154                 smix"
155                    .into(),
156            ),
157            ..Default::default()
158        }));
159    }
160    let coverage = if landed.complete {
161        ChainCoverage::Every
162    } else {
163        ChainCoverage::NamedOnly
164    };
165    // A runner that lists everything under the point and lists nothing
166    // is saying where the touch went: outside every window it can read.
167    // Measured with gesture navigation and a system dialog in front —
168    // the point was below the dialog, where no readable window reaches.
169    let verdict = if chain.is_empty() {
170        ActVerdict::Missed(format!(
171            "aimed at {} and the touch was delivered outside every window \
172             the runner can read",
173            describe_hit(&aimed)
174        ))
175    } else {
176        match (
177            tap_landed_within(&aimed, &chain, coverage),
178            &landed.uncertain,
179        ) {
180            (ActVerdict::Missed(why), Some(guess)) => ActVerdict::Unconfirmable(format!(
181                "{why} — but the runner could not be sure where the touch went: {guess}"
182            )),
183            (verdict, _) => verdict,
184        }
185    };
186    if let ActVerdict::Missed(why) = &verdict {
187        if tap_mismatch_is_fatal() {
188            return Err(ExpectationFailure::new(FailureInit {
189                code: Some(FailureCode::TapMissed),
190                message: format!(
191                    "tap did not land where it aimed: {why} ({})",
192                    describe_touch(nx, ny, landed, reader)
193                ),
194                selector: Some(selector.clone()),
195                hint: Some(
196                    "the element moved between the tree fetch and the tap, or \
197                     the touch went to something over it; wait for the screen \
198                     to settle first. Set SMIX_TAP_HIT_MISMATCH=warn to \
199                     downgrade this to a warning while migrating a suite."
200                        .into(),
201                ),
202                ..Default::default()
203            }));
204        }
205        eprintln!("smix: warning: tap did not land where it aimed: {why}");
206    }
207    Ok(ActOutcome {
208        target: Some(aimed),
209        observed: chain,
210        verdict,
211    })
212}
213
214/// Did the touch land inside the element it aimed at?
215///
216/// `chain` is every named element containing the tapped point, as the
217/// runner found them after synthesising the touch.
218///
219/// # Why containment and not identity
220///
221/// The first version of this asked whether the element at the point
222/// *was* the element aimed at. A live tree says why that is wrong. At
223/// the centre of the first row of Settings, the named elements
224/// containing the point are:
225///
226/// ```text
227/// staticText  "登录以访问iCloud数据…"                      area 7283
228/// button      id=com.apple.settings.primaryAppleAccount   area 33423
229/// application id=com.apple.Preferences                    area 351348
230/// ```
231///
232/// A flow aiming at that button taps its centre, and the innermost
233/// element there is the button's own label. Identity would call a
234/// perfectly good tap a miss — and text nested inside a row is what
235/// every list screen looks like. Containment gets it right: the button
236/// is on the chain.
237///
238/// # What the chain leaves out
239///
240/// The iOS runner lists named elements only; the Android runner lists
241/// every element under the point. `coverage` says which, and it decides
242/// what an unnamed target's absence means: nothing on iOS (it may just
243/// be unnamed), a miss on Android (the list is whole).
244///
245/// # WHAT THIS CANNOT SEE
246///
247/// **Occlusion.** A scrim covering the aimed element contains the
248/// point too, so this passes. The snapshot the runner walks carries no
249/// z-order (`TreeRoute.swift`: snapshots are dead frames), and
250/// `isHittable` — Apple's own answer — has been rejected here twice
251/// on purpose: it reports false for an element that is reachable in
252/// the AX tree but visually covered, which is exactly the see-through
253/// tap `SmixRunnerUITests.swift` performs deliberately, and it broke a
254/// QA-overlay assertion in v1.0.27.
255///
256/// So this closes the stale-frame half of "the tap reported success and
257/// nothing happened" and not the covered-element half. The whole chain
258/// travels in the outcome regardless, so a caller can see the scrim
259/// even when the verdict passes.
260pub fn tap_landed_within(
261    aimed: &HitElement,
262    chain: &[HitElement],
263    coverage: ChainCoverage,
264) -> ActVerdict {
265    if chain.is_empty() {
266        return ActVerdict::Missed(format!(
267            "aimed at {} and the tapped point held nothing — the element \
268             moved between the tree fetch and the tap, or its frame was \
269             stale",
270            describe_hit(aimed)
271        ));
272    }
273    if chain.iter().any(|c| same_element(aimed, c)) {
274        return ActVerdict::Confirmed;
275    }
276    if aimed.identifier.is_empty() && aimed.label.is_empty() && coverage == ChainCoverage::NamedOnly
277    {
278        return ActVerdict::Unconfirmable(format!(
279            "the element aimed at carries neither an identifier nor a \
280             label, so it cannot be looked for among the {} element(s) \
281             at the tapped point",
282            chain.len()
283        ));
284    }
285    ActVerdict::Missed(format!(
286        "aimed at {} but the tapped point is inside {} instead",
287        describe_hit(aimed),
288        chain
289            .iter()
290            .map(describe_hit)
291            .collect::<Vec<_>>()
292            .join(", ")
293    ))
294}
295
296/// Are these two descriptions the same element?
297///
298/// By the strongest field both carry: identifier, then label, then
299/// geometry.
300fn same_element(a: &HitElement, b: &HitElement) -> bool {
301    if !a.identifier.is_empty() && !b.identifier.is_empty() {
302        return a.identifier == b.identifier;
303    }
304    if !a.label.is_empty() && !b.label.is_empty() {
305        return a.label == b.label;
306    }
307    if a.identifier.is_empty()
308        && a.label.is_empty()
309        && b.identifier.is_empty()
310        && b.label.is_empty()
311    {
312        let close = |x: f64, y: f64| (x - y).abs() <= FRAME_TOLERANCE_PT;
313        return close(a.frame.0, b.frame.0)
314            && close(a.frame.1, b.frame.1)
315            && close(a.frame.2, b.frame.2)
316            && close(a.frame.3, b.frame.3);
317    }
318    // One is named and the other is not: they are describable in
319    // different vocabularies, which is not evidence of sameness.
320    false
321}
322
323pub(crate) fn describe_hit(e: &HitElement) -> String {
324    if !e.identifier.is_empty() {
325        format!("id={}", e.identifier)
326    } else if !e.label.is_empty() {
327        format!("label={:?}", e.label)
328    } else {
329        format!(
330            "an unnamed element at ({:.0},{:.0} {:.0}x{:.0})",
331            e.frame.0, e.frame.1, e.frame.2, e.frame.3
332        )
333    }
334}