Skip to main content

iphone_cli/
steps.rs

1//! Steps: the pieces `iph batch` runs from a JSON list, and that scripts call
2//! as `waitFor`, `readField` and `readList`.
3//!
4//! A step is a one-key map. Kinds: `launch`, `wait_for`, `tap`, `type`,
5//! `swipe`, `pinch`, `scroll_until`, `read_field`, `read_list`, `assert`, `button`,
6//! `key`, `sleep`. Any step may add `when: "a == b"` (run the
7//! step only then) and `hint: "…"` (what to tell the person if it fails). A
8//! step that fails stops the run; nothing is guessed.
9//!
10//! Strings may use today's date and the date some days ago:
11//! `{{today.day}}`, `{{ago.6.month}}`, `{{ago.6.year}}`.
12
13use std::time::{Duration, Instant};
14
15use regex::Regex;
16use serde_json::{Map, Value, json};
17
18use crate::{
19    actions::{self, ButtonArgs, KeyArgs, LaunchArgs, PinchArgs, ScrollArgs, SwipeArgs, TapArgs, TypeArgs, brief},
20    ax::{
21        attrs,
22        tree::{BuildOptions, Element, Rect, Screen},
23    },
24    device::session::{Fresh, Session, reuse},
25    fields,
26    i18n::t,
27    reply::{Fail, Res},
28    selector::{Selector, dedupe_targets},
29};
30
31const KINDS: &[&str] = &[
32    "launch",
33    "wait_for",
34    "tap",
35    "type",
36    "swipe",
37    "pinch",
38    "scroll_until",
39    "read_field",
40    "read_list",
41    "assert",
42    "button",
43    "key",
44    "sleep",
45];
46
47/// Keys a step may carry next to its kind.
48const MODIFIERS: &[&str] = &["when", "hint"];
49
50/// Split a step into (kind, body).
51fn split_step(step: &Value) -> Result<(&str, &Value), String> {
52    let map = step.as_object().ok_or("a step is a map like `tap: {...}`")?;
53    for key in ["when", "hint"] {
54        if map.get(key).is_some_and(|v| !v.is_string()) {
55            return Err(format!("`{key}` must be text"));
56        }
57    }
58    if let Some(expr) = map.get("when").and_then(|v| v.as_str()) {
59        // Only the form is checked here; the values come at run time.
60        if !expr.contains("==") && !expr.contains("!=") {
61            return Err("`when` must be `a == b` or `a != b`".into());
62        }
63    }
64    let mut kinds = map.iter().filter(|(k, _)| !MODIFIERS.contains(&k.as_str()));
65    let (kind, body) = kinds.next().ok_or("a step needs a kind")?;
66    if kinds.next().is_some() {
67        return Err("a step has exactly one kind (plus optional `when`, `hint`)".into());
68    }
69    if !KINDS.contains(&kind.as_str()) {
70        return Err(format!("unknown step kind `{kind}` ({})", KINDS.join(", ")));
71    }
72    Ok((kind.as_str(), body))
73}
74
75/// Whether a step's `when` holds: `a == b` or `a != b`, compared as text
76/// after the arguments and dates have been filled in.
77fn holds(expr: &str) -> Result<bool, String> {
78    let (left, right, equal) = match (expr.split_once("!="), expr.split_once("==")) {
79        (Some((l, r)), _) => (l, r, false),
80        (None, Some((l, r))) => (l, r, true),
81        (None, None) => return Err(format!("`when: {expr}` must be `a == b` or `a != b`")),
82    };
83    Ok((left.trim() == right.trim()) == equal)
84}
85
86/// Days since 1970-01-01 of a calendar date, and back (proleptic Gregorian).
87fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
88    let y = if m <= 2 { y - 1 } else { y };
89    let era = y.div_euclid(400);
90    let yoe = y.rem_euclid(400);
91    let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1;
92    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
93    era * 146_097 + doe - 719_468
94}
95
96fn civil_from_days(z: i64) -> (i64, i64, i64) {
97    let z = z + 719_468;
98    let era = z.div_euclid(146_097);
99    let doe = z.rem_euclid(146_097);
100    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
101    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
102    let mp = (5 * doy + 2) / 153;
103    let d = doy - (153 * mp + 2) / 5 + 1;
104    let m = if mp < 10 { mp + 3 } else { mp - 9 };
105    (yoe + era * 400 + i64::from(m <= 2), m, d)
106}
107
108/// Today in the Mac's time zone as days since 1970-01-01. `IPH_TODAY`
109/// (YYYY-MM-DD) overrides it, to try a run as if it were another day.
110fn today_days() -> i64 {
111    if let Ok(text) = std::env::var("IPH_TODAY") {
112        let parts: Vec<i64> = text.split('-').filter_map(|p| p.parse().ok()).collect();
113        if let [y, m, d] = parts[..] {
114            return days_from_civil(y, m, d);
115        }
116    }
117    let now = std::time::SystemTime::now()
118        .duration_since(std::time::UNIX_EPOCH)
119        .map(|d| d.as_secs() as libc::time_t)
120        .unwrap_or(0);
121    // SAFETY: localtime_r / localtime_s fill the struct they are given and keep nothing.
122    let tm = unsafe {
123        let mut tm: libc::tm = std::mem::zeroed();
124        #[cfg(unix)]
125        libc::localtime_r(&now, &mut tm);
126        #[cfg(windows)]
127        libc::localtime_s(&mut tm, &now);
128        tm
129    };
130    days_from_civil(i64::from(tm.tm_year) + 1900, i64::from(tm.tm_mon) + 1, i64::from(tm.tm_mday))
131}
132
133/// Fill in `{{today.day}}`, `{{ago.6.month}}` and the like.
134fn fill_dates(text: &str, today: i64) -> String {
135    static DATE: std::sync::OnceLock<Regex> = std::sync::OnceLock::new();
136    let re = DATE.get_or_init(|| {
137        Regex::new(r"\{\{\s*(?:today|ago\.(\d{1,4}))\.(year|month|day)\s*\}\}").expect("date pattern")
138    });
139    re.replace_all(text, |c: &regex::Captures| {
140        let back: i64 = c.get(1).and_then(|m| m.as_str().parse().ok()).unwrap_or(0);
141        let (y, m, d) = civil_from_days(today - back);
142        match &c[2] {
143            "year" => y.to_string(),
144            "month" => m.to_string(),
145            _ => d.to_string(),
146        }
147    })
148    .into_owned()
149}
150
151/// Replace `{{name}}` and the date expressions in every string of `v`.
152fn substitute(v: &Value, args: &Map<String, Value>, today: i64) -> Value {
153    match v {
154        Value::String(s) => {
155            let mut out = s.clone();
156            for (k, val) in args {
157                let needle = format!("{{{{{k}}}}}");
158                if out.contains(&needle) {
159                    let rep = match val {
160                        Value::String(s) => s.clone(),
161                        other => other.to_string(),
162                    };
163                    out = out.replace(&needle, &rep);
164                }
165            }
166            if out.contains("{{") {
167                out = fill_dates(&out, today);
168            }
169            Value::String(out)
170        }
171        Value::Array(a) => Value::Array(a.iter().map(|x| substitute(x, args, today)).collect()),
172        Value::Object(m) => Value::Object(
173            m.iter()
174                .map(|(k, x)| (k.clone(), substitute(x, args, today)))
175                .collect(),
176        ),
177        other => other.clone(),
178    }
179}
180
181/// A step body that is either the selector itself or `{selector: …, …}`.
182fn selector_and_options(body: &Value) -> Result<(Selector, Map<String, Value>), Fail> {
183    let (sel_value, options) = match body.as_object() {
184        Some(map) if map.contains_key("selector") => {
185            let mut rest = map.clone();
186            let sel = rest.remove("selector").unwrap();
187            (sel, rest)
188        }
189        _ => (body.clone(), Map::new()),
190    };
191    let sel = Selector::from_value(&sel_value).map_err(|e| Fail::invalid(e.to_string()))?;
192    Ok((sel, options))
193}
194
195fn opt_u64(options: &Map<String, Value>, key: &str) -> Option<u64> {
196    options.get(key).and_then(|v| v.as_u64())
197}
198
199/// Text of an element for `take`. `value` falls back to the part of a
200/// "name, value" label after the name.
201fn take(el: &Element, what: &str) -> Option<String> {
202    match what {
203        "label" => el.label.clone(),
204        "identifier" => el.identifier.clone(),
205        "text" => el.value.clone().or_else(|| el.label.clone()),
206        _ => el.value.clone().or_else(|| {
207            el.label
208                .as_deref()
209                .and_then(|l| l.split_once(", "))
210                .map(|(_, rest)| rest.to_string())
211        }),
212    }
213}
214
215fn apply_regex(text: Option<String>, pattern: Option<&str>) -> Result<Option<String>, Fail> {
216    let (Some(text), Some(pattern)) = (text.clone(), pattern) else {
217        return Ok(text);
218    };
219    let re = Regex::new(pattern).map_err(|e| Fail::invalid(format!("bad regex: {e}")))?;
220    Ok(re.captures(&text).map(|c| {
221        c.get(1)
222            .or_else(|| c.get(0))
223            .map(|m| m.as_str().to_string())
224            .unwrap_or_default()
225    }))
226}
227
228/// One list row on one reading: what is reported for it, what identifies it,
229/// and where it sat.
230#[derive(Debug, Clone)]
231struct Row {
232    value: Value,
233    key: String,
234    y: f64,
235}
236
237/// Key of the entry that stands for a group heading among the rows while a
238/// list is being collected. It never reaches the result.
239const GROUP_MARK: &str = "\u{1}group";
240/// On a row: the heading that stood above it on the reading it was taken
241/// from. Removed before the result goes out.
242const GROUP_OF: &str = "\u{1}group_of";
243
244/// Give every row the text of the heading it stands under (as column `name`)
245/// and drop the headings. With `below`, rows under a heading whose number is
246/// smaller are dropped too.
247fn settle_groups(items: Vec<Value>, name: &str, below: Option<i64>) -> Vec<Value> {
248    let mut current: Option<String> = None;
249    let mut out = Vec::new();
250    for item in items {
251        if let Some(heading) = item.get(GROUP_MARK) {
252            current = heading.as_str().map(String::from);
253            continue;
254        }
255        let mut row = item;
256        // The heading the row was seen under wins over the last heading
257        // collected: rows can come in out of order (the app put new rows in
258        // above while the list was being read), and they would otherwise be
259        // filed under whatever heading happened to come last.
260        let seen_under = match &mut row {
261            Value::Object(map) => map.remove(GROUP_OF).and_then(|v| v.as_str().map(String::from)),
262            _ => None,
263        };
264        let heading = seen_under.or_else(|| current.clone());
265        let number = heading.as_deref().and_then(|c| c.trim().parse::<i64>().ok());
266        if let (Some(limit), Some(n)) = (below, number)
267            && n < limit
268        {
269            continue;
270        }
271        if let Value::Object(map) = &mut row {
272            map.insert(name.to_string(), heading.map(Value::String).unwrap_or(Value::Null));
273        }
274        out.push(row);
275    }
276    out
277}
278
279/// Whether a heading with a number below `limit` has been collected.
280fn passed_below(items: &[Value], limit: i64) -> bool {
281    items
282        .iter()
283        .filter_map(|i| i.get(GROUP_MARK).and_then(|g| g.as_str()))
284        .filter_map(|g| g.trim().parse::<i64>().ok())
285        .any(|n| n < limit)
286}
287
288/// How far the rows moved between two readings, if they moved as one block:
289/// the distance that at least two thirds of the rows appearing once in both
290/// readings agree on. None when nothing recognisable moved, or when the rows
291/// did not move together (a list stretching at its end, a reading taken
292/// mid-motion).
293fn scroll_distance(prev: &[Row], page: &[Row]) -> Option<f64> {
294    let once = |rows: &[Row], key: &str| rows.iter().filter(|r| r.key == key).count() == 1;
295    let deltas: Vec<f64> = page
296        .iter()
297        .filter(|r| once(page, &r.key) && once(prev, &r.key))
298        .filter_map(|r| prev.iter().find(|p| p.key == r.key).map(|p| p.y - r.y))
299        .filter(|d| d.abs() > 2.0)
300        .collect();
301    // The value with the most neighbours within 3 points.
302    let (agree, distance) = deltas
303        .iter()
304        .map(|d| (deltas.iter().filter(|o| (*o - d).abs() <= 3.0).count(), *d))
305        .max_by_key(|(n, _)| *n)?;
306    (agree * 3 >= deltas.len() * 2).then_some(distance)
307}
308
309/// Rows of `page` that were not on the previous reading.
310///
311/// When the rows moved as one block, position decides: a row was there
312/// already if the same row sat where this one would have been before the
313/// scroll, or sits exactly where it sat (a pinned element, not a list row).
314/// That keeps two identical rows apart and survives pinned headers and
315/// toolbars that match the row selector.
316///
317/// A row can also be moved by something other than the scroll: more rows
318/// loading in above it, a heading growing. It is then not where the scroll
319/// would have put it and looks new, though it was there before (2026-11-11:
320/// the first row after the part of a list that had loaded came back twice).
321/// So a previous row that should still be on screen and is not at its place
322/// takes one of the "new" rows with its text off the list.
323///
324/// When they did not move as one block, only rows whose text the previous
325/// reading did not have count as new (more of the same text than before
326/// counts for the surplus). That can miss an identical neighbour, but never
327/// reports a row twice.
328fn new_rows(prev: &[Row], page: &[Row]) -> Vec<Value> {
329    fresh_rows(prev, page).into_iter().map(|r| r.value.clone()).collect()
330}
331
332/// The rows `new_rows` hands back, as they are on `page`.
333fn fresh_rows<'a>(prev: &[Row], page: &'a [Row]) -> Vec<&'a Row> {
334    match scroll_distance(prev, page) {
335        Some(dy) => {
336            let at = |r: &Row, y: f64, slack: f64| (r.y - y).abs() <= slack;
337            let mut fresh: Vec<&Row> = page
338                .iter()
339                .filter(|r| {
340                    let pinned = prev.iter().any(|p| p.key == r.key && at(r, p.y, 2.0));
341                    let seen = prev.iter().any(|p| p.key == r.key && at(r, p.y - dy, 3.0));
342                    !pinned && !seen
343                })
344                .collect();
345            let lo = page.iter().map(|r| r.y).fold(f64::INFINITY, f64::min);
346            let hi = page.iter().map(|r| r.y).fold(f64::NEG_INFINITY, f64::max);
347            for p in prev {
348                let now = p.y - dy;
349                let still_on_screen = now >= lo - 3.0 && now <= hi + 3.0;
350                let found = page.iter().any(|r| r.key == p.key && (at(r, now, 3.0) || at(r, p.y, 2.0)));
351                if !still_on_screen || found {
352                    continue;
353                }
354                let nearest = fresh
355                    .iter()
356                    .enumerate()
357                    .filter(|(_, r)| r.key == p.key)
358                    .min_by(|a, b| (a.1.y - now).abs().total_cmp(&(b.1.y - now).abs()))
359                    .map(|(i, _)| i);
360                if let Some(i) = nearest {
361                    fresh.remove(i);
362                }
363            }
364            fresh
365        }
366        None => {
367            let mut out = Vec::new();
368            for (i, r) in page.iter().enumerate() {
369                let before = prev.iter().filter(|p| p.key == r.key).count();
370                let so_far = page[..=i].iter().filter(|p| p.key == r.key).count();
371                if so_far > before {
372                    out.push(r);
373                }
374            }
375            out
376        }
377    }
378}
379
380/// Whether the rows moved against the reading direction between two
381/// readings: the list went back, so nothing on the later reading lies beyond
382/// what was already read.
383fn moved_back(prev: &[Row], page: &[Row], direction: &str) -> bool {
384    match scroll_distance(prev, page) {
385        // Reading down, rows move up the screen: y gets smaller.
386        Some(dy) if direction == "down" => dy < -3.0,
387        Some(dy) => dy > 3.0,
388        None => false,
389    }
390}
391
392/// Rows of `page` that lie beyond the end of `at_end`, the last reading
393/// before the end check (scroll back, scroll forward, read).
394///
395/// Only those can be new: a list that has really ended shows nothing past
396/// `at_end`, and one whose scrolls had not arrived shows more of itself past
397/// it. Rows on the near side of `at_end`'s last row were read on the way down
398/// already. They do appear when the forward scroll falls short of the back
399/// scroll or the reading is older than the forward scroll (2026-11-11, B7:
400/// 4 of 19 readings of Settings > General ended with two middle rows a second
401/// time, because the end check took everything not on `at_end` for new).
402///
403/// `None` when the two readings cannot be lined up (the rows did not move as
404/// one block) and the page has rows `at_end` has not: then it cannot be told
405/// which side they lie on.
406fn past_end(at_end: &[Row], page: &[Row], direction: &str) -> Option<Vec<Value>> {
407    let fresh = fresh_rows(at_end, page);
408    if fresh.is_empty() {
409        return Some(Vec::new());
410    }
411    let dy = scroll_distance(at_end, page)?;
412    // Where `at_end`'s far edge sits on this reading.
413    let edge = if direction == "down" {
414        at_end.iter().map(|r| r.y).fold(f64::NEG_INFINITY, f64::max) - dy
415    } else {
416        at_end.iter().map(|r| r.y).fold(f64::INFINITY, f64::min) - dy
417    };
418    let beyond = |r: &Row| if direction == "down" { r.y > edge + 3.0 } else { r.y < edge - 3.0 };
419    Some(fresh.into_iter().filter(|r| beyond(r)).map(|r| r.value.clone()).collect())
420}
421
422/// How far to scroll a list between two readings.
423///
424/// A fixed step (a third of the screen) reads every row two or three times.
425/// The step that reads each row once depends on the phone and the app: how
426/// tall the list's window is, how much of it stays readable under pinned
427/// headers, how much shorter the list moves than the finger. So it is worked
428/// out from what the readings show.
429#[derive(Debug, Default)]
430struct Stride {
431    /// Highest place a list item has been seen at (smallest y). Items stay
432    /// in the accessibility tree up to here, also under a pinned header.
433    top: Option<f64>,
434    /// How much shorter the list moved than the finger, on the last scroll.
435    lost: f64,
436    /// The finger's travel on the last scroll, 0 when it was a fixed step.
437    last: f64,
438}
439
440impl Stride {
441    /// Share of the screen's height the finger may cross in one scroll.
442    const REACH: f64 = 0.72;
443    /// The item brought up stops this far below the top, so that it stays
444    /// readable when the list moves a little further than meant.
445    const MARGIN: f64 = 8.0;
446
447    /// Take in what a reading shows: where items can be, and how far the
448    /// last scroll really moved them.
449    fn learn(&mut self, before: &[Row], page: &[Row]) {
450        if let Some(y) = page.iter().map(|r| r.y).reduce(f64::min) {
451            self.top = Some(self.top.map_or(y, |t| t.min(y)));
452        }
453        if self.last > 0.0
454            && let Some(moved) = scroll_distance(before, page)
455        {
456            self.lost = (self.last - moved.abs()).clamp(0.0, 200.0);
457        }
458    }
459
460    /// The finger's travel for the next scroll, or `None` when the reading
461    /// gives nothing to go by (fewer than three items) and a fixed step has
462    /// to do.
463    fn next(&mut self, page: &[Row], screen_height: f64) -> Option<f64> {
464        let top = self.top?;
465        let mut ys: Vec<f64> = page.iter().map(|r| r.y).collect();
466        ys.sort_by(f64::total_cmp);
467        if ys.len() < 3 || screen_height <= 0.0 {
468            return None;
469        }
470        // The item before the last one goes to the top: two items in common.
471        let want = ys[ys.len() - 2] - (top + Self::MARGIN);
472        // Until a scroll has been measured, count on losing little: that
473        // errs towards moving too short, which only costs overlap.
474        let lost = if self.last > 0.0 { self.lost } else { 40.0 };
475        let travel = (want + lost).min(screen_height * Self::REACH);
476        if travel < 120.0 {
477            return None;
478        }
479        self.last = travel;
480        Some(travel)
481    }
482}
483
484/// Where each row of a reading sat, for the trace: "y:mark" pairs. The mark is
485/// a short hash of the row's text, so the log shows which rows repeat or
486/// jump without holding what they say.
487fn page_marks(page: &[Row]) -> String {
488    page.iter()
489        .map(|r| {
490            // FNV-1a, 32 bits.
491            let hash = r.key.bytes().fold(0x811c_9dc5u32, |h, b| (h ^ u32::from(b)).wrapping_mul(0x0100_0193));
492            format!("{:.0}:{:06x}", r.y, hash & 0x00ff_ffff)
493        })
494        .collect::<Vec<_>>()
495        .join(" ")
496}
497
498/// Wrap a step's failure in the run's report: which step, why, and
499/// what the screen showed.
500fn step_failure(
501    index: usize,
502    raw: &Value,
503    f: Fail,
504    kind: &str,
505    records: &[Value],
506    outputs: &Map<String, Value>,
507    ms: u64,
508) -> Fail {
509    let mut extra = f.extra.clone();
510    let summary = extra.remove("screen_summary");
511    let mut out = Fail::new(
512        f.outcome,
513        "step_failed",
514        format!("step {index} ({kind}) failed: {}", f.message),
515    );
516    out.hint = f.hint.clone();
517    out = out
518        .with("failed_step", json!({ "index": index, "kind": kind, "step": raw, "elapsed_ms": ms }))
519        .with("reason", json!({ "code": f.code, "message": f.message }))
520        .with("screen_summary", summary.unwrap_or(Value::Null))
521        .with("steps", json!(records))
522        .with("outputs", json!(outputs));
523    for (k, v) in extra {
524        out.extra.entry(k).or_insert(v);
525    }
526    out
527}
528
529/// Extra fields for a step's record and, for read steps, the value read.
530type StepOutput = (Map<String, Value>, Option<(String, Value)>);
531
532impl Session {
533    pub(crate) async fn wait_for(&mut self, sel: &Selector, timeout: Duration) -> Res<Element> {
534        let started = Instant::now();
535        let full = sel.needs_identifier();
536        let mut fresh = Fresh::MaxAge(reuse());
537        let last: Option<Screen>;
538        loop {
539            let (_, screen) = self.screen(fresh, full, BuildOptions::default()).await?;
540            if let Some(el) = dedupe_targets(sel.find(&screen)).first() {
541                return Ok((*el).clone());
542            }
543            if started.elapsed() >= timeout {
544                last = Some(screen);
545                break;
546            }
547            // Each further look asks the phone again (about 1.1 s).
548            fresh = Fresh::Force;
549        }
550        Err(Fail::not_sent(
551            "wait_timeout",
552            format!("{} did not appear within {} ms", sel.describe(), timeout.as_millis()),
553        )
554        .with("screen_summary", last.map(|s| s.summary(12)).unwrap_or(Value::Null)))
555    }
556
557    /// Wait until the element a step is about to tap holds still: two
558    /// readings in a row with it at the same place. A screen that has just
559    /// come up may still be loading and move its rows; a tap aimed at where
560    /// the row was then lands on its neighbour (2026-11-11: right after a
561    /// launch a tap on one row of a home screen opened the page of the row
562    /// above it).
563    pub(crate) async fn settle_target(&mut self, sel: &Selector) -> Res<()> {
564        let full = sel.needs_identifier();
565        let place = |screen: &Screen| -> Option<(i64, i64, i64, i64)> {
566            match dedupe_targets(sel.find(screen)).as_slice() {
567                [one] => one.frame.map(|f| {
568                    (f.x.round() as i64, f.y.round() as i64, f.w.round() as i64, f.h.round() as i64)
569                }),
570                _ => None,
571            }
572        };
573        let (_, screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
574        if self.held_still(sel, &screen).await {
575            return Ok(());
576        }
577        let mut last = place(&screen);
578        for _ in 0..4 {
579            let (_, again) = self.screen(Fresh::Force, full, BuildOptions::default()).await?;
580            let now = place(&again);
581            if now.is_some() && now == last {
582                return Ok(());
583            }
584            last = now;
585        }
586        // Still moving after four looks: the tap goes ahead on the last one,
587        // as it would have without this check.
588        Ok(())
589    }
590
591    /// Whether the one element `sel` finds on `screen` is still where that
592    /// reading put it, asked of the phone directly (tens of milliseconds where
593    /// a fresh reading takes over a second). A target on the move is asked
594    /// again until two answers agree. False sends `settle_target` to full
595    /// readings: no single match, no answer, or the element is gone.
596    async fn held_still(&mut self, sel: &Selector, screen: &Screen) -> bool {
597        let el = match dedupe_targets(sel.find(screen)).as_slice() {
598            [one] => (*one).clone(),
599            _ => return false,
600        };
601        let (Some(key), Some(frame)) = (el.key, el.frame) else { return false };
602        let round = |r: &Rect| (r.x.round() as i64, r.y.round() as i64, r.w.round() as i64, r.h.round() as i64);
603        let mut last = round(&frame);
604        for _ in 0..8 {
605            let Some(ax) = self.ax.as_mut() else { return false };
606            let now = ax
607                .ask(key, attrs::FRAME, Duration::from_secs(1))
608                .await
609                .and_then(|v| v.get("NS.rectval").and_then(|r| r.as_str()).and_then(Rect::parse));
610            let Some(now) = now.map(|r| round(&r)) else { return false };
611            if now.2 == 0 && now.3 == 0 {
612                return false;
613            }
614            if now == last {
615                return true;
616            }
617            last = now;
618            tokio::time::sleep(Duration::from_millis(150)).await;
619        }
620        false
621    }
622
623    /// `name` is what a batch files the value under; a script has no use for it.
624    pub(crate) async fn read_field(&mut self, body: &Value) -> Res<(String, Value)> {
625        let map = body.as_object().ok_or_else(|| Fail::invalid("read_field needs a map"))?;
626        let name = map.get("name").and_then(|v| v.as_str()).unwrap_or("value");
627        let sel_value = map.get("selector").ok_or_else(|| Fail::invalid("read_field needs `selector`"))?;
628        let sel = Selector::from_value(sel_value).map_err(|e| Fail::invalid(e.to_string()))?;
629        let what = map.get("take").and_then(|v| v.as_str()).unwrap_or("value");
630        let pattern = map.get("regex").and_then(|v| v.as_str());
631        let timeout = Duration::from_millis(opt_u64(map, "timeout_ms").unwrap_or(5000));
632        // The element may still be on its way in.
633        Box::pin(self.wait_for(&sel, timeout)).await?;
634        let (_, screen) = self
635            .screen(Fresh::MaxAge(reuse()), sel.needs_identifier() || what == "identifier", BuildOptions::default())
636            .await?;
637        let el = actions::resolve_one(&sel, &screen)?;
638        let mut text = apply_regex(take(el, what), pattern)?;
639        // What a run hands back must not come from a reading that was
640        // already out of date: look again until two readings give the same.
641        let needs_ids = sel.needs_identifier() || what == "identifier";
642        for _ in 0..2 {
643            let (_, again) = self.screen(Fresh::Force, needs_ids, BuildOptions::default()).await?;
644            let next = match dedupe_targets(sel.find(&again)).as_slice() {
645                [one] => apply_regex(take(one, what), pattern)?,
646                _ => None,
647            };
648            let same = next == text;
649            text = next;
650            if same {
651                break;
652            }
653        }
654        let el = actions::resolve_one(&sel, &screen)?;
655        match text {
656            Some(text) => Ok((name.to_string(), Value::String(text))),
657            None => Err(Fail::not_sent(
658                "no_value",
659                format!("{} has no {what} to read", sel.describe()),
660            )
661            .with("element", brief(el))),
662        }
663    }
664
665    /// `key`: the columns that say which row a row is. Without it the whole
666    /// row does, which takes a row whose text changes while the list is read
667    /// (a clock, a counter) for a new one.
668    fn rows_on_screen(
669        screen: &Screen,
670        row: &Selector,
671        columns: Option<&Map<String, Value>>,
672        key: &[String],
673    ) -> Res<Vec<Row>> {
674        let mut out = Vec::new();
675        for el in dedupe_targets(row.find(screen)) {
676            let y = el.frame.map(|f| f.y).unwrap_or(0.0);
677            // Two rows are the same row when label and value agree.
678            let identity = format!(
679                "{}\u{1}{}",
680                el.label.as_deref().unwrap_or(""),
681                el.value.as_deref().unwrap_or("")
682            );
683            let Some(cols) = columns else {
684                let mut v = json!({});
685                if let Some(l) = &el.label {
686                    v["label"] = json!(l);
687                }
688                if let Some(val) = &el.value {
689                    v["value"] = json!(val);
690                }
691                out.push(Row { value: v, key: identity, y });
692                continue;
693            };
694            let mut rec = Map::new();
695            for (col, spec) in cols {
696                let spec_map = spec.as_object();
697                // `{value: "…"}` is the same text on every row (a month, a tag).
698                if let Some(fixed) = spec_map.and_then(|m| m.get("value")) {
699                    rec.insert(col.clone(), fixed.clone());
700                    continue;
701                }
702                let what = spec_map
703                    .and_then(|m| m.get("take"))
704                    .and_then(|v| v.as_str())
705                    .or_else(|| spec.as_str())
706                    .unwrap_or("label");
707                let pattern = spec_map.and_then(|m| m.get("regex")).and_then(|v| v.as_str());
708                // A column either reads the row itself or an element inside it.
709                let source: Option<&Element> = match spec_map.and_then(|m| m.get("selector")) {
710                    Some(inner) => {
711                        let inner = Selector::from_value(inner).map_err(|e| Fail::invalid(e.to_string()))?;
712                        let scoped = Selector {
713                            within: Some(Box::new(Selector {
714                                id: Some(crate::selector::TextMatch::Exact(el.id.clone())),
715                                ..Default::default()
716                            })),
717                            ..inner
718                        };
719                        scoped.find(screen).first().copied()
720                    }
721                    None => Some(el),
722                };
723                let text = apply_regex(source.and_then(|e| take(e, what)), pattern)?;
724                rec.insert(col.clone(), text.map(Value::String).unwrap_or(Value::Null));
725            }
726            let key = if key.is_empty() {
727                format!("{identity}\u{1}{}", Value::Object(rec.clone()))
728            } else {
729                key.iter()
730                    .map(|k| rec.get(k).map(|v| v.to_string()).unwrap_or_default())
731                    .collect::<Vec<_>>()
732                    .join("\u{1}")
733            };
734            out.push(Row { value: Value::Object(rec), key, y });
735        }
736        Ok(out)
737    }
738
739    /// The rows on one reading, with the group headings among them when the
740    /// list has groups, top to bottom.
741    fn list_page(
742        screen: &Screen,
743        row: &Selector,
744        columns: Option<&Map<String, Value>>,
745        key: &[String],
746        group: Option<&(Selector, String, Option<String>, String)>,
747    ) -> Res<Vec<Row>> {
748        let mut page = Self::rows_on_screen(screen, row, columns, key)?;
749        if let Some((sel, what, pattern, _)) = group {
750            // Headings travel with the rows, so the same position logic that
751            // keeps rows apart puts each heading where it belongs.
752            for el in dedupe_targets(sel.find(screen)) {
753                let Some(text) = apply_regex(take(el, what), pattern.as_deref())? else {
754                    continue;
755                };
756                page.push(Row {
757                    key: format!("{GROUP_MARK}\u{1}{}", el.label.as_deref().unwrap_or("")),
758                    value: json!({ GROUP_MARK: text }),
759                    y: el.frame.map(|f| f.y).unwrap_or(0.0),
760                });
761            }
762            page.sort_by(|a, b| a.y.total_cmp(&b.y));
763            // Note on every row which heading stands above it on this reading.
764            let mut above: Option<Value> = None;
765            for item in page.iter_mut() {
766                if let Some(heading) = item.value.get(GROUP_MARK) {
767                    above = Some(heading.clone());
768                } else if let (Some(heading), Value::Object(map)) = (&above, &mut item.value) {
769                    map.insert(GROUP_OF.to_string(), heading.clone());
770                }
771            }
772        }
773        Ok(page)
774    }
775
776    /// Drag the list by `travel` points of finger movement, slowing to a
777    /// stop. "down" shows what is further down.
778    async fn stride_drag(&mut self, direction: &str, travel: f64) -> Res<bool> {
779        let (_, screen) = self.screen(Fresh::MaxAge(reuse()), false, BuildOptions::default()).await?;
780        let (w, h) = (screen.width, screen.height);
781        let (low, high) = (h * 0.88, h * 0.88 - travel);
782        let (from, to) = if direction == "down" { (low, high) } else { (high, low) };
783        // A longer way takes the finger longer, as a hand's would.
784        let duration = Duration::from_millis((350.0 + travel * 0.6) as u64);
785        let (changed, _) = Box::pin(self.drag_points(
786            (w / 2.0, from),
787            (w / 2.0, to),
788            duration,
789            crate::device::input::Motion::Place,
790        ))
791        .await?;
792        Ok(changed)
793    }
794
795    /// One scroll step of a list; whether the screen moved.
796    async fn list_scroll(&mut self, direction: &str) -> Res<bool> {
797        let r = Box::pin(self.cmd_scroll(ScrollArgs {
798            direction: direction.to_string(),
799            until: None,
800            max: None,
801        }))
802        .await?;
803        Ok(r.get("screen_changed").and_then(|v| v.as_bool()).unwrap_or(false))
804    }
805
806    pub(crate) async fn read_list(&mut self, body: &Value) -> Res<(String, Value, Value)> {
807        let map = body.as_object().ok_or_else(|| Fail::invalid("read_list needs a map"))?;
808        let name = map.get("name").and_then(|v| v.as_str()).unwrap_or("rows");
809        let row = Selector::from_value(map.get("row").ok_or_else(|| Fail::invalid("read_list needs `row`"))?)
810            .map_err(|e| Fail::invalid(e.to_string()))?;
811        let columns = map.get("columns").and_then(|v| v.as_object());
812        // `key: "city"` (or a list): the columns that identify a row.
813        let key: Vec<String> = match map.get("key") {
814            None | Some(Value::Null) => Vec::new(),
815            Some(Value::String(k)) => vec![k.clone()],
816            Some(Value::Array(ks)) => ks
817                .iter()
818                .map(|k| k.as_str().map(String::from).ok_or_else(|| Fail::invalid("key takes column names")))
819                .collect::<Res<_>>()?,
820            Some(_) => return Err(Fail::invalid("key takes a column name or a list of them")),
821        };
822        if let Some(missing) = key.iter().find(|k| !columns.is_some_and(|c| c.contains_key(k.as_str()))) {
823            return Err(Fail::invalid(format!("key names `{missing}`, which is not one of the columns")));
824        }
825        let until = match map.get("until") {
826            Some(v) => Some(Selector::from_value(v).map_err(|e| Fail::invalid(e.to_string()))?),
827            None => None,
828        };
829        let max_scrolls = opt_u64(map, "max_scrolls").unwrap_or(15).min(200) as u32;
830        let limit = opt_u64(map, "limit").unwrap_or(500) as usize;
831        let direction = map.get("direction").and_then(|v| v.as_str()).unwrap_or("down").to_string();
832        // `group: {selector, take, regex, as}`: headings between the rows (a
833        // date above each day's rows). Every row gets the heading it stands
834        // under as column `as`.
835        let group = match map.get("group") {
836            Some(Value::Object(g)) => {
837                let sel = Selector::from_value(g.get("selector").ok_or_else(|| Fail::invalid("group needs `selector`"))?)
838                    .map_err(|e| Fail::invalid(e.to_string()))?;
839                let what = g.get("take").and_then(|v| v.as_str()).unwrap_or("label").to_string();
840                let pattern = g.get("regex").and_then(|v| v.as_str()).map(String::from);
841                let name = g.get("as").and_then(|v| v.as_str()).unwrap_or("group").to_string();
842                Some((sel, what, pattern, name))
843            }
844            Some(_) => return Err(Fail::invalid("group must be a map: {selector, take, regex, as}")),
845            None => None,
846        };
847        // `stop_below: N`: stop once a heading whose number is below N has
848        // come up, and leave its rows out (a list that runs back in time,
849        // read down to a day of the month).
850        let stop_below = match map.get("stop_below") {
851            None | Some(Value::Null) => None,
852            Some(v) => Some(
853                v.as_i64()
854                    .or_else(|| v.as_str().and_then(|s| s.trim().parse().ok()))
855                    .ok_or_else(|| Fail::invalid("stop_below must be a whole number"))?,
856            ),
857        };
858        if stop_below.is_some() && group.is_none() {
859            return Err(Fail::invalid("stop_below needs `group`"));
860        }
861        let full = row.needs_identifier()
862            || until.as_ref().is_some_and(|u| u.needs_identifier())
863            || group.as_ref().is_some_and(|g| g.0.needs_identifier());
864
865        let mut all: Vec<Value> = Vec::new();
866        let mut prev: Option<Vec<Row>> = None;
867        let mut scrolls = 0u32;
868        // Readings in a row that brought no new row.
869        let mut barren = 0u32;
870        // Whether the end of the list has been checked by scrolling back and forth.
871        let mut end_checked = false;
872        let back = if direction == "down" { "up" } else { "down" };
873        let mut stride = Stride::default();
874        let ended;
875        loop {
876            // One reading a page: the scroll before it already asked the
877            // phone for the new screen, and this takes that reading over.
878            let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
879            let mut page = Self::list_page(&screen, &row, columns, &key, group.as_ref())?;
880            if let Some(before) = &prev {
881                let stitched = page.iter().any(|r| before.iter().any(|p| p.key == r.key));
882                if !stitched && !page.is_empty() && !before.is_empty() && stride.last > 0.0 {
883                    // The scroll carried the list past everything that was on
884                    // screen, so rows may lie unseen in between. Come back
885                    // half way and read again.
886                    crate::device::ax::trace(|| "read_list: no row in common after the scroll; coming back half way".into());
887                    self.stride_drag(back, stride.last / 2.0).await?;
888                    stride.lost += 40.0;
889                    (_, screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
890                    page = Self::list_page(&screen, &row, columns, &key, group.as_ref())?;
891                }
892                stride.learn(before, &page);
893            } else {
894                stride.learn(&[], &page);
895            }
896            match &prev {
897                None => all.extend(page.iter().map(|r| r.value.clone())),
898                Some(before) if moved_back(before, &page, &direction) => {
899                    // A reading from before the scroll arrived, or the list
900                    // bounced back: the rows on it were read already. Keep
901                    // the furthest reading to compare the next one with, or
902                    // rows read before would come back as new.
903                    crate::device::ax::trace(|| format!("read_list page: moved back [{}]", page_marks(&page)));
904                    barren += 1;
905                    if until.as_ref().is_some_and(|u| !u.find(&screen).is_empty()) {
906                        ended = "until";
907                        break;
908                    }
909                    if scrolls >= max_scrolls {
910                        ended = "max_scrolls";
911                        break;
912                    }
913                    self.list_scroll(&direction).await?;
914                    stride.last = 0.0;
915                    scrolls += 1;
916                    continue;
917                }
918                Some(before) => {
919                    let fresh_rows = new_rows(before, &page);
920                    crate::device::ax::trace(|| {
921                        format!(
922                            "read_list page: rows={} moved={:?} new={} [{}]",
923                            page.len(),
924                            scroll_distance(before, &page),
925                            fresh_rows.len(),
926                            page_marks(&page),
927                        )
928                    });
929                    barren = if fresh_rows.is_empty() { barren + 1 } else { 0 };
930                    all.extend(fresh_rows);
931                }
932            }
933            prev = Some(page);
934            if stop_below.is_some_and(|n| passed_below(&all, n)) {
935                ended = "stop_below";
936                break;
937            }
938            if all.iter().filter(|i| i.get(GROUP_MARK).is_none()).count() >= limit {
939                ended = "limit";
940                break;
941            }
942            if until.as_ref().is_some_and(|u| !u.find(&screen).is_empty()) {
943                ended = "until";
944                break;
945            }
946            if barren >= 2 {
947                // Two readings without a new row. That is the end of the list,
948                // or scrolls that never arrived: an input session the phone
949                // has stopped listening to looks exactly the same, and a list
950                // cut short there reads as complete (2026-11-11: two scrolls
951                // right after a Face ID prompt moved nothing, and a week of
952                // rows came back as two days). So the end is checked once by
953                // scrolling back and forward: at a real end the list moves
954                // back and returns to the same rows.
955                if end_checked {
956                    ended = "end_of_list";
957                    break;
958                }
959                end_checked = true;
960                let at_end = prev.clone().unwrap_or_default();
961                let mut moved = self.list_scroll(back).await?;
962                if !moved {
963                    // Nothing moves either way: once more on a new input session.
964                    self.close_input().await;
965                    moved = self.list_scroll(back).await?;
966                }
967                if !moved {
968                    // A list that fits the screen.
969                    ended = "end_of_list";
970                    break;
971                }
972                self.list_scroll(&direction).await?;
973                let mut fresh = Fresh::MaxAge(reuse());
974                let mut looks = 0;
975                let (page, fresh_rows) = loop {
976                    let (_, screen) = self.screen(fresh, full, BuildOptions::default()).await?;
977                    let page = Self::list_page(&screen, &row, columns, &key, group.as_ref())?;
978                    let past = past_end(&at_end, &page, &direction);
979                    crate::device::ax::trace(|| {
980                        format!(
981                            "read_list end check: moved={:?} new={:?} beyond={:?} [{}]",
982                            scroll_distance(&at_end, &page),
983                            new_rows(&at_end, &page).len(),
984                            past.as_ref().map(|p| p.len()),
985                            page_marks(&page),
986                        )
987                    });
988                    match past {
989                        Some(rows) => break (page, rows),
990                        // Rows the end did not have, but not lined up with
991                        // it: the reading may be mid-motion. Look again.
992                        None if looks < 2 => {
993                            looks += 1;
994                            fresh = Fresh::Force;
995                        }
996                        None => {
997                            return Err(Fail::unknown(
998                                "list_uncertain",
999                                "at the end of the list, the screen could not be lined up with what was read; rows may be missing or repeated",
1000                            )
1001                            .hint(t(
1002                                "목록을 끝까지 읽었는지 확신할 수 없어 결과를 내지 않았습니다. 다시 실행하세요.",
1003                                "Not sure the whole list was read, so no result was given. Run it again.",
1004                            ))
1005                            .with("rows_so_far", all.len())
1006                            .with("scrolls", scrolls));
1007                        }
1008                    }
1009                };
1010                if fresh_rows.is_empty() {
1011                    ended = "end_of_list";
1012                    break;
1013                }
1014                all.extend(fresh_rows);
1015                barren = 0;
1016                prev = Some(page);
1017                if stop_below.is_some_and(|n| passed_below(&all, n)) {
1018                    ended = "stop_below";
1019                    break;
1020                }
1021            }
1022            if scrolls >= max_scrolls {
1023                ended = "max_scrolls";
1024                break;
1025            }
1026            // Scroll by as much as the screen allows: the last two items of
1027            // this reading go to the top of the next, so that the two
1028            // readings can be stitched and nothing in between is skipped.
1029            match stride.next(prev.as_deref().unwrap_or(&[]), screen.height) {
1030                Some(travel) => {
1031                    self.stride_drag(&direction, travel).await?;
1032                }
1033                None => {
1034                    self.list_scroll(&direction).await?;
1035                    stride.last = 0.0;
1036                }
1037            }
1038            scrolls += 1;
1039        }
1040        let mut rows = match &group {
1041            Some((_, _, _, column)) => settle_groups(all, column, stop_below),
1042            None => all,
1043        };
1044        rows.truncate(limit);
1045        Ok((
1046            name.to_string(),
1047            Value::Array(rows),
1048            json!({ "scrolls": scrolls, "ended": ended }),
1049        ))
1050    }
1051
1052    /// Reconnection discards the old accessibility subscription. Re-read before
1053    /// using a remembered launch guard, so it cannot silently become a no-op.
1054    pub(crate) async fn check_expected_app(&mut self, expected_pid: Option<i64>) -> Res<()> {
1055        self.refresh_connection();
1056        if expected_pid.is_some() && self.ax.as_ref().is_none_or(|ax| ax.is_closed()) {
1057            Box::pin(self.cmd_read(Default::default())).await?;
1058        }
1059        match self.unexpected_app(expected_pid) {
1060            Some(f) => Err(f),
1061            None => Ok(()),
1062        }
1063    }
1064
1065    /// A failure when the latest reading shows another app in front than the
1066    /// one this run launched (a notification was tapped, a link opened …).
1067    pub(crate) fn unexpected_app(&self, expected_pid: Option<i64>) -> Option<Fail> {
1068        let expected = expected_pid?;
1069        let (_, dump, _) = self.ax.as_ref()?.latest()?;
1070        if dump.applications.iter().any(|a| a.pid == expected) {
1071            return None;
1072        }
1073        let screen = Screen::build(&dump, BuildOptions::default());
1074        Some(
1075            Fail::not_sent(
1076                "unexpected_app",
1077                format!(
1078                    "another app came to the front: {}",
1079                    screen
1080                        .apps
1081                        .iter()
1082                        .filter_map(|a| a.name.clone())
1083                        .collect::<Vec<_>>()
1084                        .join(", ")
1085                ),
1086            )
1087            .hint(t(
1088                "알림 배너를 눌렀거나 다른 앱이 열렸을 수 있습니다. 화면을 확인하세요. 집중 모드(방해금지)를 켜 두면 배너가 뜨지 않습니다.",
1089                "A notification banner may have been tapped or another app opened. Check the screen. A Focus (Do Not Disturb) keeps banners away.",
1090            ))
1091            .with("screen_summary", screen.summary(6)),
1092        )
1093    }
1094
1095    /// Run one step. Returns extra fields for the step's record and, for read
1096    /// steps, the value read.
1097    async fn run_step(&mut self, kind: &str, body: &Value) -> Res<StepOutput> {
1098        fn no_value(f: Map<String, Value>) -> Res<StepOutput> {
1099            Ok((f, None))
1100        }
1101        match kind {
1102            "launch" => {
1103                let args: LaunchArgs = match body {
1104                    Value::String(s) => LaunchArgs { app: s.clone(), ..Default::default() },
1105                    other => actions::parse_args(other)?,
1106                };
1107                let app = args.app.clone();
1108                let r = Box::pin(self.cmd_launch(args)).await?;
1109                if r.get("foreground").and_then(|v| v.as_bool()) != Some(true) {
1110                    return Err(Fail::unknown(
1111                        "launch_not_verified",
1112                        format!("{app} was launched but is not in front"),
1113                    ));
1114                }
1115                no_value(fields! {
1116                    "bundle_id": r.get("bundle_id").cloned().unwrap_or(Value::Null),
1117                    "pid": r.get("pid").cloned().unwrap_or(Value::Null),
1118                })
1119            }
1120            "wait_for" => {
1121                let (sel, options) = selector_and_options(body)?;
1122                let timeout = Duration::from_millis(opt_u64(&options, "timeout_ms").unwrap_or(8000));
1123                let el = Box::pin(self.wait_for(&sel, timeout)).await?;
1124                no_value(fields! { "element": brief(&el) })
1125            }
1126            "assert" => {
1127                let (sel, _) = selector_and_options(body)?;
1128                // One patient look: the screen may be a moment behind.
1129                let el = Box::pin(self.wait_for(&sel, Duration::from_millis(1500))).await.map_err(|mut f| {
1130                    f.code = "assert_failed".into();
1131                    f.message = format!("expected {} on screen", sel.describe());
1132                    f
1133                })?;
1134                no_value(fields! { "element": brief(&el) })
1135            }
1136            "tap" if body.is_array() => {
1137                let r = Box::pin(self.tap_sequence(body.as_array().unwrap())).await?;
1138                no_value(fields! {
1139                    "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null),
1140                    "taps": r.get("taps").cloned().unwrap_or(Value::Null),
1141                })
1142            }
1143            "tap" => {
1144                let (sel, options) = selector_and_options(body)?;
1145                // Give the target a moment to appear, as a person would.
1146                let timeout = Duration::from_millis(opt_u64(&options, "timeout_ms").unwrap_or(5000));
1147                Box::pin(self.wait_for(&sel, timeout)).await?;
1148                if options.get("settle").and_then(|v| v.as_bool()).unwrap_or(true) {
1149                    Box::pin(self.settle_target(&sel)).await?;
1150                }
1151                let target = match body.as_object() {
1152                    Some(m) if m.contains_key("selector") => m["selector"].clone(),
1153                    _ => body.clone(),
1154                };
1155                let r = Box::pin(self.cmd_tap(TapArgs {
1156                    target,
1157                    long_ms: opt_u64(&options, "long_ms"),
1158                    double: options.get("double").and_then(|v| v.as_bool()).unwrap_or(false),
1159                    no_wait: options.get("no_wait").and_then(|v| v.as_bool()).unwrap_or(false),
1160                }))
1161                .await?;
1162                no_value(fields! {
1163                    "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null),
1164                    "element": r.get("element").cloned().unwrap_or(Value::Null),
1165                })
1166            }
1167            "type" => {
1168                let args: TypeArgs = match body {
1169                    Value::String(s) => TypeArgs { text: s.clone(), ..Default::default() },
1170                    other => actions::parse_args(other)?,
1171                };
1172                let r = Box::pin(self.cmd_type(args)).await?;
1173                no_value(r)
1174            }
1175            "swipe" => {
1176                let args: SwipeArgs = match body {
1177                    Value::String(s) => SwipeArgs { direction: Some(s.clone()), ..Default::default() },
1178                    other => actions::parse_args(other)?,
1179                };
1180                let r = Box::pin(self.cmd_swipe(args)).await?;
1181                no_value(fields! { "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null) })
1182            }
1183            "pinch" => {
1184                let args: PinchArgs = match body {
1185                    Value::String(s) => PinchArgs { direction: Some(s.clone()), ..Default::default() },
1186                    other => actions::parse_args(other)?,
1187                };
1188                let r = Box::pin(self.cmd_pinch(args)).await?;
1189                no_value(fields! { "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null) })
1190            }
1191            "scroll_until" => {
1192                let (_, options) = selector_and_options(body)?;
1193                let until = match body.as_object() {
1194                    Some(m) if m.contains_key("selector") => m["selector"].clone(),
1195                    _ => body.clone(),
1196                };
1197                let r = Box::pin(self.cmd_scroll(ScrollArgs {
1198                    direction: options
1199                        .get("direction")
1200                        .and_then(|v| v.as_str())
1201                        .unwrap_or("down")
1202                        .to_string(),
1203                    until: Some(until),
1204                    max: opt_u64(&options, "max").map(|n| n as u32),
1205                }))
1206                .await?;
1207                no_value(fields! {
1208                    "scrolls": r.get("scrolls").cloned().unwrap_or(Value::Null),
1209                    "element": r.get("element").cloned().unwrap_or(Value::Null),
1210                })
1211            }
1212            "read_field" => {
1213                let (name, value) = Box::pin(self.read_field(body)).await?;
1214                Ok((fields! { "name": name.clone(), "value": value.clone() }, Some((name, value))))
1215            }
1216            "read_list" => {
1217                let (name, rows, info) = Box::pin(self.read_list(body)).await?;
1218                let count = rows.as_array().map(|a| a.len()).unwrap_or(0);
1219                Ok((
1220                    fields! { "name": name.clone(), "count": count, "scrolls": info["scrolls"], "ended": info["ended"] },
1221                    Some((name, rows)),
1222                ))
1223            }
1224            "button" => {
1225                let button = body.as_str().ok_or_else(|| Fail::invalid("button needs a name"))?;
1226                let r = Box::pin(self.cmd_button(ButtonArgs { button: button.to_string() })).await?;
1227                no_value(fields! { "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null) })
1228            }
1229            "key" => {
1230                let key = body.as_str().ok_or_else(|| Fail::invalid("key needs a name"))?;
1231                let r = Box::pin(self.cmd_key(KeyArgs { key: key.to_string() })).await?;
1232                no_value(fields! { "screen_changed": r.get("screen_changed").cloned().unwrap_or(Value::Null) })
1233            }
1234            "sleep" => {
1235                let ms = body.as_u64().ok_or_else(|| Fail::invalid("sleep needs milliseconds"))?;
1236                tokio::time::sleep(Duration::from_millis(ms.min(60_000))).await;
1237                no_value(Map::new())
1238            }
1239            other => Err(Fail::invalid(format!("unknown step kind `{other}`"))),
1240        }
1241    }
1242
1243    /// Run steps in order and stop at the first failure.
1244    pub async fn run_steps(&mut self, steps: &[Value], args: &Map<String, Value>) -> Res {
1245        let mut records: Vec<Value> = Vec::new();
1246        let mut outputs = Map::new();
1247        // The app the last `launch` step brought to the front. Every later
1248        // step must still find it there.
1249        let mut expected_pid: Option<i64> = None;
1250        // Steps that sent input, or may have, including the failing step.
1251        let mut inputs_sent = 0u32;
1252        let today = today_days();
1253        for (i, raw) in steps.iter().enumerate() {
1254            let index = i + 1;
1255            let started = Instant::now();
1256            let step = substitute(raw, args, today);
1257            let (kind, body) = match split_step(&step) {
1258                Ok(x) => x,
1259                Err(e) => return Err(step_failure(index, raw, Fail::invalid(e), "?", &records, &outputs, 0)),
1260            };
1261            if let Some(expr) = step.get("when").and_then(|v| v.as_str()) {
1262                match holds(expr) {
1263                    Ok(true) => {}
1264                    Ok(false) => {
1265                        records.push(json!({ "index": index, "kind": kind, "ok": true, "skipped": true, "when": expr }));
1266                        continue;
1267                    }
1268                    Err(e) => return Err(step_failure(index, raw, Fail::invalid(e), kind, &records, &outputs, 0)),
1269                }
1270            }
1271            // A `wait_for` may sit out something the system puts over the app
1272            // for a moment (a Face ID prompt); every other step needs the app
1273            // in front.
1274            if kind != "wait_for"
1275                && let Err(f) = Box::pin(self.check_expected_app(expected_pid)).await
1276            {
1277                return Err(step_failure(index, raw, f, kind, &records, &outputs, 0));
1278            }
1279            let hint = step.get("hint").and_then(|v| v.as_str()).map(String::from);
1280            // Boxed: the step futures are large and would otherwise pile up on
1281            // the stack of whoever polls the run.
1282            let result = Box::pin(self.run_step(kind, body)).await;
1283            let ms = started.elapsed().as_millis() as u64;
1284            match result {
1285                Ok((extra, value)) => {
1286                    if kind == "launch" {
1287                        expected_pid = extra.get("pid").and_then(|v| v.as_i64());
1288                    }
1289                    if matches!(
1290                        kind,
1291                        "launch" | "tap" | "type" | "swipe" | "pinch" | "scroll_until" | "read_list" | "button" | "key"
1292                    ) {
1293                        inputs_sent += 1;
1294                    }
1295                    let mut rec = fields! { "index": index, "kind": kind, "ok": true, "elapsed_ms": ms };
1296                    for (k, v) in extra {
1297                        if !v.is_null() || kind == "type" {
1298                            rec.insert(k, v);
1299                        }
1300                    }
1301                    records.push(Value::Object(rec));
1302                    if let Some((name, value)) = value {
1303                        outputs.insert(name, value);
1304                    }
1305                }
1306                Err(mut f) => {
1307                    if f.outcome != crate::reply::Outcome::NotSent {
1308                        inputs_sent += 1;
1309                    }
1310                    // Whoever wrote the steps knows what this one usually waits on.
1311                    if let Some(hint) = hint {
1312                        f.hint = Some(hint);
1313                    }
1314                    return Err(step_failure(index, raw, f, kind, &records, &outputs, ms)
1315                        .with("inputs_sent", inputs_sent));
1316                }
1317            }
1318        }
1319        if let Some(f) = self.unexpected_app(expected_pid) {
1320            return Err(step_failure(steps.len(), &Value::Null, f, "end", &records, &outputs, 0));
1321        }
1322        Ok(fields! { "steps": records, "outputs": outputs })
1323    }
1324}
1325
1326#[cfg(test)]
1327mod tests {
1328    use super::*;
1329
1330    #[test]
1331    fn rejects_bad_steps() {
1332        assert!(split_step(&json!({"fly": "away"})).unwrap_err().contains("unknown step kind"));
1333        assert!(split_step(&json!({"tap": "a", "type": "b"})).unwrap_err().contains("exactly one kind"));
1334        assert!(split_step(&json!({"tap": "a", "hint": "look"})).is_ok());
1335        assert!(split_step(&json!({"tap": "a", "confirm": true})).is_err());
1336    }
1337
1338    #[test]
1339    fn a_pinch_step_is_a_kind_and_takes_a_direction_or_its_options() {
1340        for step in [json!({"pinch": "out"}), json!({"pinch": {"scale": 1.5, "rotate": 30, "at": "Map", "duration_ms": 300}})] {
1341            let (kind, body) = split_step(&step).unwrap();
1342            assert_eq!(kind, "pinch");
1343            if body.is_object() {
1344                assert!(actions::parse_args::<PinchArgs>(body).is_ok());
1345            }
1346        }
1347        let unknown = actions::parse_args::<PinchArgs>(&json!({"zoom": 2})).unwrap_err();
1348        assert_eq!((unknown.code.as_str(), unknown.outcome), ("invalid_argument", crate::reply::Outcome::NotSent));
1349    }
1350
1351    #[test]
1352    fn substitutes_arguments() {
1353        let step = json!({"type": {"text": "{{a}}×{{b}}"}});
1354        let mut args = Map::new();
1355        args.insert("a".into(), json!("12"));
1356        args.insert("b".into(), json!(34));
1357        assert_eq!(substitute(&step, &args, 0)["type"]["text"], "12×34");
1358    }
1359
1360    #[test]
1361    fn dates_fill_in_and_cross_months_and_years() {
1362        // 2026-11-03: six days earlier is 28 October.
1363        let day = days_from_civil(2026, 11, 3);
1364        assert_eq!(civil_from_days(day), (2026, 11, 3));
1365        assert_eq!(fill_dates("{{today.month}}/{{today.day}}", day), "11/3");
1366        assert_eq!(fill_dates("{{ago.6.month}}/{{ago.6.day}}", day), "10/28");
1367        assert_eq!(fill_dates("{{ ago.0.year }}", day), "2026");
1368        // 2027-01-02: six days earlier is in the year before.
1369        let day = days_from_civil(2027, 1, 2);
1370        assert_eq!(fill_dates("{{ago.6.year}}-{{ago.6.month}}-{{ago.6.day}}", day), "2026-12-27");
1371        // A leap day.
1372        assert_eq!(civil_from_days(days_from_civil(2028, 3, 1) - 1), (2028, 2, 29));
1373        // Anything else in braces is left for the arguments.
1374        assert_eq!(fill_dates("{{name}} {{today.week}}", day), "{{name}} {{today.week}}");
1375        let step = json!({"read_list": {"stop_below": "{{ago.6.day}}"}, "when": "{{ago.6.month}} != {{today.month}}"});
1376        let filled = substitute(&step, &Map::new(), days_from_civil(2026, 11, 3));
1377        assert_eq!(filled["read_list"]["stop_below"], "28");
1378        assert_eq!(filled["when"], "10 != 11");
1379    }
1380
1381    #[test]
1382    fn when_compares_text() {
1383        assert!(holds("11 == 11").unwrap());
1384        assert!(!holds("10 == 11").unwrap());
1385        assert!(holds("10 != 11").unwrap());
1386        assert!(!holds(" 11 !=11 ").unwrap());
1387        assert!(holds("soon").is_err());
1388        assert!(split_step(&json!({"tap": "a", "when": "maybe"})).unwrap_err().contains("when"));
1389        assert!(split_step(&json!({"tap": "a", "when": "{{ago.6.month}} != {{today.month}}"})).is_ok());
1390    }
1391
1392    /// A reading of Settings > General: rows `from` onwards, the first at
1393    /// `top`, one every 52 points down to the bottom of the screen.
1394    fn general(from: &str, top: f64) -> Vec<Row> {
1395        const ROWS: [&str; 24] = [
1396            "설정", "정보", "소프트웨어 업데이트", "iPhone 저장 공간", "AppleCare 및 보증", "AirDrop",
1397            "AirPlay 및 연속성", "화면 속 화면", "화면 캡처", "CarPlay", "게임 컨트롤러", "날짜 및 시간",
1398            "백그라운드 앱 새로 고침", "사전", "서체", "언어 및 지역", "자동 완성 및 암호", "키보드",
1399            "트랙패드", "TV 제공업체", "VPN 및 기기 관리", "법률 및 규제 정보", "전송 또는 iPhone 재설정",
1400            "시스템 종료",
1401        ];
1402        let start = ROWS.iter().position(|r| *r == from).unwrap();
1403        ROWS[start..]
1404            .iter()
1405            .enumerate()
1406            .map(|(i, label)| (label, top + 52.0 * i as f64))
1407            .take_while(|(_, y)| *y < 880.0)
1408            .map(|(label, y)| Row { value: json!({ "label": label }), key: label.to_string(), y })
1409            .collect()
1410    }
1411
1412    #[test]
1413    fn the_end_check_does_not_take_rows_read_on_the_way_down() {
1414        // B7 (2026-11-11), 4 of 19: at the end of the list, the scroll back
1415        // and the scroll forward did not cancel out, and the reading after
1416        // them stood 104 points short of the end. Two rows above the end's
1417        // first row came back as new.
1418        let at_end = general("서체", 120.0);
1419        let short = general("백그라운드 앱 새로 고침", 120.0);
1420        assert_eq!(scroll_distance(&at_end, &short), Some(-104.0));
1421        assert_eq!(labels(&new_rows(&at_end, &short)), ["백그라운드 앱 새로 고침", "사전"]);
1422        assert!(moved_back(&at_end, &short, "down"));
1423        assert_eq!(past_end(&at_end, &short, "down"), Some(vec![]));
1424        // Back at the same place: nothing new either.
1425        assert_eq!(past_end(&at_end, &at_end, "down"), Some(vec![]));
1426    }
1427
1428    #[test]
1429    fn the_end_check_takes_rows_past_the_end() {
1430        // The scrolls before the check had not arrived: the list moves on
1431        // and shows rows past the last reading, which are new.
1432        let stuck = general("AirDrop", 120.0);
1433        let further = general("날짜 및 시간", 120.0);
1434        assert_eq!(stuck.last().unwrap().key, "TV 제공업체");
1435        let past = past_end(&stuck, &further, "down").unwrap();
1436        assert_eq!(
1437            labels(&past),
1438            ["VPN 및 기기 관리", "법률 및 규제 정보", "전송 또는 iPhone 재설정", "시스템 종료"]
1439        );
1440        assert!(!moved_back(&stuck, &further, "down"));
1441    }
1442
1443    #[test]
1444    fn the_end_check_gives_up_on_a_reading_it_cannot_line_up() {
1445        // Rows in common but each at a different offset (a reading taken
1446        // mid-motion), and two rows the end did not have.
1447        let at_end = general("서체", 120.0);
1448        let mut smeared = general("백그라운드 앱 새로 고침", 120.0);
1449        for (i, r) in smeared.iter_mut().enumerate() {
1450            r.y += 17.0 * i as f64;
1451        }
1452        assert_eq!(scroll_distance(&at_end, &smeared), None);
1453        assert_eq!(past_end(&at_end, &smeared, "down"), None);
1454    }
1455
1456    #[test]
1457    fn reading_up_the_list_mirrors_reading_down() {
1458        // Reading upward, the list's end is its top.
1459        let at_top = general("설정", 120.0);
1460        let short = general("화면 속 화면", 120.0);
1461        assert!(moved_back(&at_top, &short, "up"));
1462        assert_eq!(past_end(&at_top, &short, "up"), Some(vec![]));
1463    }
1464
1465    fn mark(text: &str) -> Value {
1466        json!({ GROUP_MARK: text })
1467    }
1468
1469    #[test]
1470    fn rows_take_the_heading_they_stand_under() {
1471        let items = vec![
1472            json!({"label": "before any heading"}),
1473            mark("11"),
1474            json!({"label": "a"}),
1475            json!({"label": "b"}),
1476            mark("9"),
1477            json!({"label": "c"}),
1478            mark("4"),
1479            json!({"label": "d"}),
1480        ];
1481        let rows = settle_groups(items.clone(), "day", None);
1482        let days: Vec<Value> = rows.iter().map(|r| r["day"].clone()).collect();
1483        assert_eq!(days, [Value::Null, json!("11"), json!("11"), json!("9"), json!("4")]);
1484        // Reading down to day 5: the rows of day 4 are left out.
1485        assert!(!passed_below(&items[..6], 5));
1486        assert!(passed_below(&items, 5));
1487        let rows = settle_groups(items, "day", Some(5));
1488        assert_eq!(labels(&rows), ["before any heading", "a", "b", "c"]);
1489    }
1490
1491    #[test]
1492    fn a_row_keeps_the_heading_it_was_seen_under() {
1493        // Day 11's rows were collected, then the heading of day 10 and its
1494        // row. Then the app put three new rows in at the top of day 11; they
1495        // arrive last, but were seen under "11".
1496        let late = |label: &str| json!({ "label": label, GROUP_OF: "11" });
1497        let items = vec![
1498            mark("11"),
1499            json!({"label": "a"}),
1500            mark("10"),
1501            json!({"label": "b"}),
1502            late("new 1"),
1503            late("new 2"),
1504        ];
1505        let rows = settle_groups(items, "day", None);
1506        let got: Vec<(&str, &str)> = rows.iter().map(|r| (r["day"].as_str().unwrap(), r["label"].as_str().unwrap())).collect();
1507        assert_eq!(got, [("11", "a"), ("10", "b"), ("11", "new 1"), ("11", "new 2")]);
1508        assert!(rows.iter().all(|r| r.get(GROUP_OF).is_none()));
1509    }
1510
1511    #[test]
1512    fn headings_move_with_the_rows_from_one_reading_to_the_next() {
1513        let heading = |text: &str, y: f64| Row { value: mark(text), key: format!("{GROUP_MARK}\u{1}{text}"), y };
1514        let row = |label: &str, y: f64| Row { value: json!({ "label": label }), key: label.to_string(), y };
1515        // Two days with the same transfer on each; the list moved up by 130.
1516        let before = vec![heading("11", 100.0), row("이체 100,000원", 140.0), row("커피", 205.0), heading("10", 270.0)];
1517        let after = vec![row("커피", 75.0), heading("10", 140.0), row("이체 100,000원", 180.0), heading("9", 245.0)];
1518        let mut all: Vec<Value> = before.iter().map(|r| r.value.clone()).collect();
1519        all.extend(new_rows(&before, &after));
1520        let rows = settle_groups(all, "day", None);
1521        let got: Vec<(String, String)> = rows
1522            .iter()
1523            .map(|r| (r["day"].as_str().unwrap().to_string(), r["label"].as_str().unwrap().to_string()))
1524            .collect();
1525        assert_eq!(
1526            got,
1527            [
1528                ("11".to_string(), "이체 100,000원".to_string()),
1529                ("11".to_string(), "커피".to_string()),
1530                ("10".to_string(), "이체 100,000원".to_string()),
1531            ]
1532        );
1533    }
1534
1535    fn rows(items: &[(&str, f64)]) -> Vec<Row> {
1536        items
1537            .iter()
1538            .map(|(label, y)| Row { value: json!({ "label": label }), key: label.to_string(), y: *y })
1539            .collect()
1540    }
1541
1542    fn labels(v: &[Value]) -> Vec<&str> {
1543        v.iter().map(|x| x["label"].as_str().unwrap()).collect()
1544    }
1545
1546    #[test]
1547    fn only_rows_that_were_not_on_the_previous_reading_are_new() {
1548        // The list moved up by 160 points.
1549        let before = rows(&[("a", 100.0), ("b", 180.0), ("c", 260.0), ("d", 340.0)]);
1550        let after = rows(&[("c", 100.0), ("d", 180.0), ("e", 260.0), ("f", 340.0)]);
1551        assert_eq!(scroll_distance(&before, &after), Some(160.0));
1552        assert_eq!(labels(&new_rows(&before, &after)), ["e", "f"]);
1553    }
1554
1555    #[test]
1556    fn pinned_elements_that_match_the_row_selector_are_counted_once() {
1557        // "합계" stays at the top and "받아쓰기" at the bottom on every reading.
1558        let before = rows(&[("합계", 60.0), ("a", 100.0), ("b", 180.0), ("받아쓰기", 860.0)]);
1559        let after = rows(&[("합계", 60.0), ("b", 100.0), ("c", 180.0), ("받아쓰기", 860.0)]);
1560        assert_eq!(labels(&new_rows(&before, &after)), ["c"]);
1561    }
1562
1563    #[test]
1564    fn identical_rows_are_kept_apart_by_position() {
1565        // Three purchases that read the same; the list moved up by 80.
1566        let before = rows(&[("커피 4,500원", 100.0), ("커피 4,500원", 180.0), ("빵 3,000원", 260.0)]);
1567        let after = rows(&[
1568            ("커피 4,500원", 100.0),
1569            ("빵 3,000원", 180.0),
1570            ("커피 4,500원", 260.0),
1571            ("우유 2,000원", 340.0),
1572        ]);
1573        // The first two are the old second and third rows; the coffee at 260
1574        // and the milk are new.
1575        assert_eq!(labels(&new_rows(&before, &after)), ["커피 4,500원", "우유 2,000원"]);
1576    }
1577
1578    #[test]
1579    fn the_stride_brings_the_last_two_items_to_the_top() {
1580        let mut stride = Stride::default();
1581        // First reading: the list starts half way down the screen.
1582        let first = rows(&[("a", 500.0), ("b", 565.0), ("c", 630.0), ("d", 695.0), ("e", 760.0), ("f", 825.0)]);
1583        stride.learn(&[], &first);
1584        // "e" (the one before the last) should go to the top (500 + margin);
1585        // nothing measured yet, so 40 is added for what the list lags.
1586        let travel = stride.next(&first, 932.0).unwrap();
1587        assert_eq!(travel, 760.0 - 508.0 + 40.0);
1588        // The list moved 232 for a travel of 292: it lags by 60.
1589        let second = rows(&[("e", 528.0), ("f", 593.0), ("g", 658.0), ("h", 723.0), ("i", 788.0), ("j", 853.0)]);
1590        stride.learn(&first, &second);
1591        assert_eq!(stride.lost, 60.0);
1592        assert_eq!(stride.top, Some(500.0));
1593        // Next: "i" to the top, with the lag that was measured.
1594        assert_eq!(stride.next(&second, 932.0).unwrap(), 788.0 - 508.0 + 60.0);
1595        // Never further than the finger can go on this screen.
1596        let far = rows(&[("x", 100.0), ("y", 800.0), ("z", 900.0)]);
1597        stride.top = Some(100.0);
1598        assert_eq!(stride.next(&far, 932.0).unwrap(), 932.0 * Stride::REACH);
1599        // Too little on screen to go by: a fixed step instead.
1600        assert_eq!(stride.next(&rows(&[("a", 500.0), ("b", 565.0)]), 932.0), None);
1601    }
1602
1603    #[test]
1604    fn a_row_pushed_down_by_rows_loading_above_it_is_not_new() {
1605        // The list moved up by 270. Meanwhile more of it loaded, which pushed
1606        // the last row 40 further down than the scroll alone would have.
1607        let before = rows(&[("a", 400.0), ("b", 465.0), ("c", 530.0), ("d", 595.0), ("last loaded", 660.0)]);
1608        let after = rows(&[
1609            ("a", 130.0),
1610            ("b", 195.0),
1611            ("c", 260.0),
1612            ("d", 325.0),
1613            ("last loaded", 430.0),
1614            ("next", 495.0),
1615        ]);
1616        assert_eq!(scroll_distance(&before, &after), Some(270.0));
1617        assert_eq!(labels(&new_rows(&before, &after)), ["next"]);
1618        // Two rows that read the same, one of them pushed: the other is new.
1619        let before = rows(&[("a", 400.0), ("b", 465.0), ("c", 530.0), ("커피", 595.0)]);
1620        let after = rows(&[("a", 130.0), ("b", 195.0), ("c", 260.0), ("커피", 365.0), ("커피", 430.0)]);
1621        assert_eq!(new_rows(&before, &after).len(), 1);
1622    }
1623
1624    #[test]
1625    fn rows_that_did_not_move_together_are_matched_by_text() {
1626        // The end of a list stretching under the finger: every row shifted by
1627        // a different amount. Nothing is new.
1628        let before = rows(&[("a", 94.0), ("b", 143.0), ("c", 193.0), ("d", 242.0), ("e", 326.0)]);
1629        let after = rows(&[("a", 114.0), ("b", 173.0), ("c", 238.0), ("d", 302.0), ("e", 406.0)]);
1630        assert_eq!(scroll_distance(&before, &after), None);
1631        assert!(new_rows(&before, &after).is_empty());
1632        // The same, with one row that was not there before.
1633        let after = rows(&[("b", 173.0), ("c", 238.0), ("d", 302.0), ("e", 406.0), ("f", 470.0)]);
1634        assert_eq!(labels(&new_rows(&before, &after)), ["f"]);
1635    }
1636
1637    #[test]
1638    fn a_reading_with_nothing_in_common_is_all_new() {
1639        let before = rows(&[("a", 100.0), ("b", 180.0)]);
1640        let after = rows(&[("x", 100.0), ("y", 180.0)]);
1641        assert_eq!(scroll_distance(&before, &after), None);
1642        assert_eq!(labels(&new_rows(&before, &after)), ["x", "y"]);
1643    }
1644
1645    #[test]
1646    fn a_key_column_keeps_a_row_whose_text_changed_from_counting_twice() {
1647        // A world clock: the minute turns between two readings of the list.
1648        let screen = |minute: u32, top: f64, cities: &[&str]| -> Screen {
1649            let elements: Vec<Value> = cities
1650                .iter()
1651                .enumerate()
1652                .map(|(i, city)| {
1653                    json!({
1654                        "id": format!("row{i}{minute}"), "role": "text", "traits": ["static_text"],
1655                        "label": format!("{city}, 오후 2:{minute:02}, 오늘, +0시간"),
1656                        "frame": { "x": 0, "y": top + 90.0 * i as f64, "w": 430, "h": 90 },
1657                    })
1658                })
1659                .collect();
1660            serde_json::from_value(json!({ "width": 430.0, "height": 932.0, "apps": [], "elements": elements })).unwrap()
1661        };
1662        let row = Selector::from_value(&json!({ "role": "text" })).unwrap();
1663        let columns = json!({
1664            "city": { "take": "label", "regex": "^([^,]+)" },
1665            "time": { "take": "label", "regex": "(오후 [0-9:]+)" },
1666        });
1667        let columns = columns.as_object();
1668        let before = screen(3, 200.0, &["서울", "도쿄", "런던", "파리"]);
1669        let after = screen(4, 20.0, &["런던", "파리", "뉴욕"]);
1670        let cities = |key: &[String]| -> Vec<String> {
1671            let prev = Session::rows_on_screen(&before, &row, columns, key).unwrap();
1672            let page = Session::rows_on_screen(&after, &row, columns, key).unwrap();
1673            new_rows(&prev, &page).iter().map(|r| r["city"].as_str().unwrap().to_string()).collect()
1674        };
1675        // By the whole row, the two that were there already look new.
1676        assert_eq!(cities(&[]), ["런던", "파리", "뉴욕"]);
1677        assert_eq!(cities(&["city".to_string()]), ["뉴욕"]);
1678    }
1679
1680    #[test]
1681    fn take_value_falls_back_to_the_label_remainder() {
1682        let mut el = Element {
1683            id: "a".into(),
1684            role: "button".into(),
1685            label: Some("iOS 버전, 27.0.1".into()),
1686            value: None,
1687            identifier: None,
1688            hint: None,
1689            frame: None,
1690            traits: vec![],
1691            enabled: true,
1692            selected: false,
1693            focused: false,
1694            parent: None,
1695            depth: 1,
1696            pid: 1,
1697            on_screen: true,
1698            raw: None,
1699            key: None,
1700        };
1701        assert_eq!(take(&el, "value").as_deref(), Some("27.0.1"));
1702        assert_eq!(take(&el, "label").as_deref(), Some("iOS 버전, 27.0.1"));
1703        el.value = Some("실제 값".into());
1704        assert_eq!(take(&el, "value").as_deref(), Some("실제 값"));
1705        assert_eq!(
1706            apply_regex(Some("합계 12,300원".into()), Some(r"([\d,]+)원")).unwrap().as_deref(),
1707            Some("12,300")
1708        );
1709    }
1710}