Skip to main content

iphone_cli/
actions.rs

1//! The commands themselves, shared by the CLI, the MCP server, `batch` and
2//! scripts. Each takes JSON arguments and returns the fields of its result.
3
4use std::{
5    collections::BTreeMap,
6    time::{Duration, Instant},
7};
8
9use serde::Deserialize;
10use serde_json::{Value, json};
11use unicode_normalization::UnicodeNormalization;
12
13use crate::{
14    ax::{
15        attrs,
16        dump::ObjKey,
17        tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
18    },
19    device::{
20        ax::trace,
21        input::{Motion, normalize},
22        pinch,
23        session::{Fresh, Gesture, Session, reuse},
24    },
25    fields,
26    i18n::t,
27    reply::{Fail, Res},
28    selector::{Selector, dedupe_targets},
29};
30
31/// How long after an input to ask for the new screen. Measured on iOS 27: a
32/// request made while a navigation animation still runs comes back about
33/// 0.4 s later than one made just after it (2026-11-10, 14 taps).
34pub const AFTER_TAP: Duration = Duration::from_millis(480);
35/// A drag that ends with the finger at rest only leaves a short bounce.
36pub const AFTER_DRAG: Duration = Duration::from_millis(300);
37pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
38pub const AFTER_PASTE: Duration = Duration::from_millis(200);
39/// How long a tap waits for the screen to start changing. A picker that
40/// another part of the app puts up came about 1.5 s after the tap
41/// (2026-11-12, 당근 '이미지 추가').
42pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
43/// How long after the tap a changed screen may take to hold still.
44pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
45/// A reading up to this old is used for a tap once the phone confirms, by a
46/// few quick questions, that the screen is still as read. Older ones are read
47/// again in full.
48const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
49/// How long a pasted text may take to show in its field.
50const PASTE_SHOWS: Duration = Duration::from_millis(1500);
51/// How long to wait for the answer to one quick question.
52const ASK_WAIT: Duration = Duration::from_millis(1000);
53/// An unasked dump this recently before an input means the app is one that
54/// triggers them; its readings are then double-checked.
55const UNASKED_MEMORY: Duration = Duration::from_secs(5);
56/// The result's note for an input sent while the screen could not be read.
57const UNREAD_NOTE: &str =
58    "the screen could not be read; sent on the last known screen size, not verified";
59
60#[derive(Debug, Default, Deserialize)]
61#[serde(default, deny_unknown_fields)]
62pub struct ReadArgs {
63    /// Only the foreground application's elements.
64    pub app: bool,
65    /// Include elements outside the screen.
66    pub all: bool,
67    /// Wait for the second pass so identifiers are present.
68    pub full: bool,
69    /// Attach every numbered attribute.
70    pub raw: bool,
71    /// Accept a cached reading up to this old instead of asking the phone.
72    pub max_age_ms: Option<u64>,
73}
74
75#[derive(Debug, Deserialize)]
76#[serde(deny_unknown_fields)]
77pub struct FindArgs {
78    pub selector: Value,
79    #[serde(default)]
80    pub all: bool,
81}
82
83#[derive(Debug, Default, Deserialize)]
84#[serde(default, deny_unknown_fields)]
85pub struct TapArgs {
86    /// A selector (string or object), `"x,y"`, or `{x, y}`.
87    pub target: Value,
88    pub long_ms: Option<u64>,
89    pub double: bool,
90    /// Read the screen once right after the tap instead of waiting for it to
91    /// change and hold still.
92    pub no_wait: bool,
93}
94
95#[derive(Debug, Default, Deserialize)]
96#[serde(default, deny_unknown_fields)]
97pub struct SwipeArgs {
98    pub direction: Option<String>,
99    pub from: Option<[f64; 2]>,
100    pub to: Option<[f64; 2]>,
101    pub duration_ms: Option<u64>,
102    /// Slow to a stop before lifting (a drag) instead of flicking.
103    pub drag: bool,
104}
105
106#[derive(Debug, Default, Deserialize)]
107#[serde(default, deny_unknown_fields)]
108pub struct PinchArgs {
109    /// `in` (scale 0.5) or `out` (scale 2); not with `scale`.
110    pub direction: Option<String>,
111    /// End span over start span, 0.25..4.
112    pub scale: Option<f64>,
113    /// Degrees, positive clockwise, -180..180.
114    pub rotate: Option<f64>,
115    /// Where the fingers meet: a selector, `"x,y"`, or `{x, y}`. Default: the
116    /// middle of the screen.
117    pub at: Value,
118    pub duration_ms: Option<u64>,
119}
120
121impl PinchArgs {
122    /// The `(scale, rotate)` these arguments ask for.
123    fn motion(&self) -> Res<(f64, f64)> {
124        let scale = match (self.direction.as_deref(), self.scale) {
125            (Some(_), Some(_)) => return Err(Fail::invalid("a direction and a scale cannot be combined")),
126            (Some("in"), None) => 0.5,
127            (Some("out"), None) => 2.0,
128            (Some(other), None) => {
129                return Err(Fail::invalid(format!("unknown direction `{other}` (in, out)")));
130            }
131            (None, scale) => scale.unwrap_or(1.0),
132        };
133        let rotate = self.rotate.unwrap_or(0.0);
134        pinch::check_motion(scale, rotate).map_err(Fail::invalid)?;
135        Ok((scale, rotate))
136    }
137}
138
139#[derive(Debug, Default, Deserialize)]
140#[serde(default, deny_unknown_fields)]
141pub struct ScrollArgs {
142    pub direction: String,
143    pub until: Option<Value>,
144    pub max: Option<u32>,
145}
146
147#[derive(Debug, Default, Deserialize)]
148#[serde(default, deny_unknown_fields)]
149pub struct TypeArgs {
150    pub text: String,
151    pub clear: bool,
152    pub submit: bool,
153}
154
155/// Observations before Return. Reading a value is distinct from seeing it
156/// change, and neither proves that it equals the caller's intended text.
157struct TypeObservation {
158    before: Option<String>,
159    value: Option<String>,
160    changed: Option<bool>,
161    warning: Option<Value>,
162}
163
164impl TypeObservation {
165    fn new(args: &TypeArgs, before: Option<String>, value: Option<String>) -> Self {
166        let changed = before.as_ref().zip(value.as_ref()).map(|(a, b)| a != b);
167        let retry_hint = t(
168            "입력이 반영되었을 수 있습니다. 자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요.",
169            "Input may have taken effect. Do not automatically retype; check with `iph read` or a screenshot.",
170        );
171        let warning = if value.is_none() {
172            Some(json!({
173                "code": "value_unreadable",
174                "message": "the input was sent but the field's value could not be read afterwards",
175                "hint": retry_hint,
176            }))
177        } else if args.text.is_empty() && value.as_deref() == Some("") {
178            // Clearing an already empty field is an observable no-op.
179            None
180        } else if args.text.is_empty() {
181            Some(json!({
182                "code": "clear_not_observed",
183                "message": "the clear input was sent but the field still shows a nonempty value",
184                "hint": retry_hint,
185            }))
186        } else if changed == Some(false) {
187            Some(json!({
188                "code": "value_unchanged",
189                "message": "the field's value is unchanged; the paste may not have taken effect (identical replacement, masking or app behavior can also cause this)",
190                "hint": t(
191                    "자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요. 붙여넣기가 막혔을 수도 있습니다: 설정 → 앱 → 해당 앱 → 다른 앱에서 붙여넣기 → 허용. 값이 같다는 사실만으로 권한 거부를 확정할 수는 없습니다.",
192                    "Do not automatically retype; check with `iph read` or a screenshot. Pasting may be blocked: Settings → Apps → this app → Paste from Other Apps → Allow. An unchanged value alone does not prove permission was denied.",
193                ),
194            }))
195        } else if value.as_deref() == Some("") {
196            Some(json!({
197                "code": "value_empty",
198                "message": "the input was sent but the field shows an empty value; the app may have cleared it",
199                "hint": retry_hint,
200            }))
201        } else if changed.is_none() {
202            Some(json!({
203                "code": "value_change_unknown",
204                "message": "the field's value was read afterwards, but its previous value was unavailable for comparison",
205                "hint": retry_hint,
206            }))
207        } else {
208            None
209        };
210        Self {
211            before,
212            value,
213            changed,
214            warning,
215        }
216    }
217
218    fn can_submit(&self) -> bool {
219        self.warning.is_none()
220    }
221
222    fn result(
223        self,
224        args: &TypeArgs,
225        field: &Element,
226        restored: bool,
227        submitted: Option<bool>,
228        screen_changed: Option<bool>,
229        submit_error: Option<String>,
230    ) -> Res {
231        let blocked_submit = args.submit && !self.can_submit();
232        let mut out = fields! {
233            "sent": true,
234            "verified": self.value.is_some(),
235            "value_before": self.before,
236            "value": self.value,
237            "value_changed": self.changed,
238            "field": brief(field),
239            "submit_sent": submitted,
240            "clipboard_restored": restored,
241            "screen_changed": screen_changed,
242        };
243        if let Some(warning) = self.warning {
244            out.insert("warning".into(), warning);
245        }
246        let fail = if let Some(e) = submit_error {
247            Some(Fail::unknown("input_failed", e))
248        } else if blocked_submit {
249            Some(Fail::unknown(
250                "type_not_verified",
251                "the input was sent, but its effect is uncertain; Return was not sent",
252            ))
253        } else {
254            None
255        };
256        if let Some(mut fail) = fail {
257            fail.hint = Some(t(
258                "입력은 이미 나갔습니다. 자동으로 재입력하거나 제출하지 말고 `iph read`나 스크린샷으로 먼저 확인하세요.",
259                "Input has already been sent. Do not automatically retype or submit; check with `iph read` or a screenshot first.",
260            ).into());
261            fail.extra = out;
262            Err(fail)
263        } else {
264            Ok(out)
265        }
266    }
267}
268
269#[derive(Debug, Deserialize)]
270#[serde(deny_unknown_fields)]
271pub struct KeyArgs {
272    pub key: String,
273}
274
275#[derive(Debug, Deserialize)]
276#[serde(deny_unknown_fields)]
277pub struct ButtonArgs {
278    pub button: String,
279}
280
281#[derive(Debug, Default, Deserialize)]
282#[serde(deny_unknown_fields)]
283pub struct LaunchArgs {
284    pub app: String,
285    /// Quit the app first, so it starts from its first screen.
286    #[serde(default)]
287    pub restart: bool,
288    /// Passed to the app's process, as after `--` in `devicectl device process launch`.
289    #[serde(default)]
290    pub arguments: Vec<String>,
291    #[serde(default)]
292    pub environment: BTreeMap<String, String>,
293}
294
295impl LaunchArgs {
296    pub fn validate(&self) -> Res<()> {
297        for key in self.environment.keys() {
298            if key.is_empty() || key.contains('=') {
299                return Err(Fail::invalid(format!(
300                    "environment names must be non-empty and contain no `=`: \"{key}\""
301                )));
302            }
303        }
304        Ok(())
305    }
306
307    /// Arguments and environment only reach a process that starts now.
308    pub fn has_launch_options(&self) -> bool {
309        !self.arguments.is_empty() || !self.environment.is_empty()
310    }
311
312    pub fn kill_existing(&self) -> bool {
313        self.restart || self.has_launch_options()
314    }
315
316    fn environment_dictionary(&self) -> Option<plist::Dictionary> {
317        (!self.environment.is_empty()).then(|| {
318            self.environment
319                .iter()
320                .map(|(k, v)| (k.clone(), plist::Value::String(v.clone())))
321                .collect()
322        })
323    }
324}
325
326#[derive(Debug, Deserialize)]
327#[serde(deny_unknown_fields)]
328pub struct ScreenshotArgs {
329    pub path: String,
330}
331
332pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
333    let v = if args.is_null() { json!({}) } else { args.clone() };
334    serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
335}
336
337pub fn brief(el: &Element) -> Value {
338    let mut v = json!({ "id": el.id, "role": el.role });
339    if let Some(l) = &el.label {
340        v["label"] = json!(l);
341    }
342    if let Some(val) = &el.value {
343        v["value"] = json!(val);
344    }
345    if let Some(i) = &el.identifier {
346        v["identifier"] = json!(i);
347    }
348    if let Some(f) = &el.frame {
349        v["frame"] = json!(f);
350    }
351    v
352}
353
354pub enum Target {
355    Selector(Selector),
356    Point(f64, f64),
357}
358
359/// The reading failed because the phone's accessibility daemon gave none on a
360/// phone not known to be locked (`Session::unread_size`).
361fn daemon_silent(read_failed: &Fail) -> bool {
362    let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
363    read_failed.code == "ax_timeout" && !locked
364}
365
366pub fn parse_target(v: &Value) -> Res<Target> {
367    if let Some(obj) = v.as_object()
368        && let (Some(x), Some(y)) = (
369            obj.get("x").and_then(|n| n.as_f64()),
370            obj.get("y").and_then(|n| n.as_f64()),
371        )
372            && obj.len() == 2 {
373                return Ok(Target::Point(x, y));
374            }
375    if let Some(s) = v.as_str() {
376        let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
377        if parts.len() == 2
378            && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
379                return Ok(Target::Point(x, y));
380            }
381    }
382    Selector::from_value(v)
383        .map(Target::Selector)
384        .map_err(|e| Fail::invalid(e.to_string()))
385}
386
387/// The single element a selector names, or an error that lists what matched.
388pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
389    let matches = dedupe_targets(sel.find(screen));
390    match matches.len() {
391        1 => Ok(matches[0]),
392        0 => Err(Fail::not_sent(
393            "not_found",
394            format!("no element matches {}", sel.describe()),
395        )
396        .hint(t(
397            "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
398            "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
399        ))
400        .with("selector", sel.describe())
401        .with("screen_summary", screen.summary(12))),
402        n => Err(Fail::not_sent(
403            "ambiguous_selector",
404            format!("{n} elements match {}", sel.describe()),
405        )
406        .hint(t(
407            "role=, within=, index= 으로 하나만 남게 좁히세요.",
408            "Narrow it to one with role=, within= or index=.",
409        ))
410        .with("selector", sel.describe())
411        .with(
412            "candidates",
413            Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
414        )),
415    }
416}
417
418/// A point inside the visible part of `el` that no other element covers.
419fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
420    let frame = el.frame.ok_or_else(|| {
421        Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
422    })?;
423    let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
424    let visible = frame.intersection(&bounds).ok_or_else(|| {
425        Fail::not_sent("off_screen", "the element is outside the screen")
426            .hint(t(
427                "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
428                "Bring it on screen with `iph scroll down --until …`.",
429            ))
430            .with("element", brief(el))
431    })?;
432    let covered = |x: f64, y: f64| -> Option<&Element> {
433        let hit = screen.element_at(x, y)?;
434        if hit.id == el.id {
435            return None;
436        }
437        // A child of the target (a label inside a row) is fine; an element
438        // that merely overlaps it (a toolbar over a list) is not.
439        let inside = hit.frame.is_some_and(|h| {
440            h.x >= frame.x - 1.0
441                && h.y >= frame.y - 1.0
442                && h.x + h.w <= frame.x + frame.w + 1.0
443                && h.y + h.h <= frame.y + frame.h + 1.0
444        });
445        let encloses = hit.frame.is_some_and(|h| {
446            frame.x >= h.x - 1.0
447                && frame.y >= h.y - 1.0
448                && frame.x + frame.w <= h.x + h.w + 1.0
449                && frame.y + frame.h <= h.y + h.h + 1.0
450        });
451        (!inside && !encloses).then_some(hit)
452    };
453    let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
454    let mut blocker = None;
455    for (fx, fy) in candidates {
456        let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
457        match covered(x, y) {
458            None => return Ok((x, y)),
459            Some(b) => blocker = blocker.or(Some(b)),
460        }
461    }
462    Err(Fail::not_sent("covered", "another element covers the target")
463        .hint(t(
464            "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
465            "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
466        ))
467        .with("element", brief(el))
468        .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
469}
470
471fn nfc(s: &str) -> String {
472    s.nfc().collect()
473}
474
475/// The app now in front, when it is not the one that was in front before.
476fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
477    let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
478    let now = after.apps.first()?;
479    (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
480}
481
482impl Session {
483    fn build_opts(all: bool, raw: bool) -> BuildOptions {
484        BuildOptions { include_offscreen: all, raw }
485    }
486
487    pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
488        let fresh = match args.max_age_ms {
489            Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
490            None => Fresh::Force,
491        };
492        let (seq, mut screen) = self
493            .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
494            .await?;
495        if args.app
496            && let Some(front) = screen.apps.first().map(|a| a.pid) {
497                screen.elements.retain(|e| e.pid == front);
498                screen.apps.retain(|a| a.pid == front);
499            }
500        let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
501        Ok(fields! {
502            "screen": {"width": screen.width, "height": screen.height},
503            "apps": screen.apps,
504            "count": screen.elements.len(),
505            "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
506            "seq": seq,
507            "elements": screen.elements,
508        })
509    }
510
511    pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
512        let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
513        let (_, screen) = self
514            .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
515            .await?;
516        let matches = sel.find(&screen);
517        let mut out = fields! {
518            "selector": sel.describe(),
519            "count": matches.len(),
520            "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
521        };
522        if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
523            out.insert(
524                "note".into(),
525                json!("this reading came without identifiers, so id= could not match; try again"),
526            );
527        }
528        Ok(out)
529    }
530
531    /// Read the screen after an input that began at `since`: wait `delay` for
532    /// the screen to react, then ask the phone for a new dump.
533    ///
534    /// Some apps make the phone send dumps unasked while they change
535    /// (Calculator on every key). Such a dump shows the screen as it was a
536    /// moment earlier and can arrive late enough to pass for the answer; the
537    /// calculator example then read "12×3" instead of "408". So with `careful`
538    /// (inputs that change the screen several times), or when the phone has
539    /// lately sent unasked dumps, the screen is read until two readings in a
540    /// row agree.
541    pub(crate) async fn observe_with(
542        &mut self,
543        since: Instant,
544        delay: Duration,
545        careful: bool,
546    ) -> Option<(u64, Screen)> {
547        tokio::time::sleep(delay).await;
548        let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
549        let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
550        let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
551        if !careful && !chatty {
552            return Some(latest);
553        }
554        for _ in 0..2 {
555            trace(|| "observe: reading again until two readings agree".into());
556            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
557                break;
558            };
559            let same = next.1.signature() == latest.1.signature();
560            latest = next;
561            if same {
562                break;
563            }
564        }
565        Some(latest)
566    }
567
568    pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
569        self.observe_with(since, delay, false).await
570    }
571
572    /// Read the screen after an input until it has changed from `before` and
573    /// then holds still, as a browser waits for a page to load. The phone
574    /// sends no event when a screen comes up, so "came up" is two readings in
575    /// a row that agree. A screen that does not change within `CHANGE_WAIT`,
576    /// or keeps moving until `SETTLE_WAIT`, comes back as it is then.
577    pub(crate) async fn observe_settled(
578        &mut self,
579        since: Instant,
580        before: &[ElementSignature],
581    ) -> Option<(u64, Screen)> {
582        let mut latest = self.observe(since, AFTER_TAP).await?;
583        while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
584            trace(|| "observe: the screen has not changed yet; reading again".into());
585            match self.screen(Fresh::Force, false, BuildOptions::default()).await {
586                Ok(next) => latest = next,
587                Err(_) => return Some(latest),
588            }
589        }
590        if latest.1.signature() == before {
591            return Some(latest);
592        }
593        while since.elapsed() < SETTLE_WAIT {
594            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
595                break;
596            };
597            let same = next.1.signature() == latest.1.signature();
598            latest = next;
599            if same {
600                break;
601            }
602            trace(|| "observe: the screen is still moving; reading again".into());
603        }
604        Some(latest)
605    }
606
607    /// Read the screen and find where to tap for each target. Fails without
608    /// sending anything when a target is missing, ambiguous or covered.
609    async fn locate(
610        &mut self,
611        targets: &[Target],
612        fresh: Fresh,
613    ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
614        let need_ids = targets
615            .iter()
616            .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
617        let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
618        if screen.width <= 0.0 || screen.height <= 0.0 {
619            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
620        }
621        let mut points = Vec::with_capacity(targets.len());
622        for target in targets {
623            points.push(match target {
624                Target::Selector(sel) => {
625                    let el = resolve_one(sel, &screen)?;
626                    let (x, y) = tap_point(el, &screen)?;
627                    (x, y, Some(el.clone()))
628                }
629                Target::Point(x, y) => {
630                    if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
631                        return Err(Fail::invalid(format!(
632                            "point {x},{y} is outside the {}x{} screen",
633                            screen.width, screen.height
634                        )));
635                    }
636                    (*x, *y, screen.element_at(*x, *y).cloned())
637                }
638            });
639        }
640        Ok((screen, points))
641    }
642
643    /// Locate the targets on the latest reading when it is too old to reuse
644    /// as it is (an agent's turn takes seconds) but the phone confirms it:
645    /// every application still shows the same visible children (nothing came
646    /// up or went away) and every target is still where it was. Each question
647    /// takes tens of milliseconds; a full reading takes over a second. `None`
648    /// sends the caller to a full reading.
649    async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
650        let age = self.reading_age()?;
651        if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
652            return None;
653        }
654        let (_, dump, _) = self.ax.as_ref()?.latest()?;
655        let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
656        let ax = self.ax.as_mut()?;
657        for app in &dump.applications {
658            let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
659            let now: Vec<ObjKey> = ax
660                .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
661                .await?
662                .as_array()?
663                .iter()
664                .filter_map(ObjKey::from_ka)
665                .collect();
666            if now != was {
667                trace(|| "locate: the screen has changed since the last reading".into());
668                return None;
669            }
670        }
671        for (_, _, el) in &located.1 {
672            let el = el.as_ref()?;
673            let now = ax
674                .ask(el.key?, attrs::FRAME, ASK_WAIT)
675                .await?
676                .get("NS.rectval")
677                .and_then(|v| v.as_str())
678                .and_then(Rect::parse)?;
679            let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
680            if el.frame.as_ref().map(round) != Some(round(&now)) {
681                trace(|| "locate: a target has moved since the last reading".into());
682                return None;
683            }
684        }
685        trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
686        Some(located)
687    }
688
689    /// Locate the targets, make sure input can be sent, and locate them again
690    /// if getting ready took long enough for the screen to have moved on: a
691    /// cold input session takes seconds to open, and a tap must never go to
692    /// where something used to be.
693    async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
694        let located = match self.locate_confirmed(targets).await {
695            Some(l) => l,
696            None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
697        };
698        let started = std::time::Instant::now();
699        self.ensure_input().await?;
700        if started.elapsed() < Duration::from_millis(400) {
701            return Ok(located);
702        }
703        self.locate(targets, Fresh::Force).await
704    }
705
706    pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
707        if args.double && args.long_ms.is_some() {
708            return Err(Fail::invalid("--double and --long cannot be combined"));
709        }
710        if let Some(ms) = args.long_ms
711            && !(1..=60_000).contains(&ms)
712        {
713            return Err(Fail::invalid("--long must be 1..60000 ms"));
714        }
715        let target = parse_target(&args.target)?;
716        let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
717            Ok(located) => located,
718            Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
719        };
720        let (x, y, element) = points.remove(0);
721        let element_json = element.as_ref().map(brief);
722
723        let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
724        let before = screen.signature();
725        let input_began = Instant::now();
726        let gesture = if args.double {
727            Gesture::DoubleTap { x: nx, y: ny }
728        } else {
729            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
730        };
731        self.send_gesture(gesture).await?;
732        let after = if args.no_wait {
733            self.observe(input_began, AFTER_TAP).await
734        } else {
735            self.observe_settled(input_began, &before).await
736        };
737        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
738        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
739        if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
740            // Something of the system's is over the app now.
741            self.input_suspect = true;
742        }
743
744        let mut out = fields! {
745            "sent": true,
746            "verified": changed.is_some(),
747            "screen_changed": changed.unwrap_or(false),
748            "x": (x * 10.0).round() / 10.0,
749            "y": (y * 10.0).round() / 10.0,
750        };
751        if let Some(e) = element_json {
752            out.insert("element".into(), e);
753        }
754        if changed.is_none() {
755            out.insert(
756                "note".into(),
757                json!("the tap was sent but no reading came back to compare"),
758            );
759        }
760        if let Some(change) = app_change {
761            out.insert("app_changed".into(), json!(true));
762            out.insert("app".into(), change);
763        }
764        // The screen it waited for, so that no reading has to follow.
765        if !args.no_wait
766            && let Some((seq, screen)) = after {
767                out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
768                out.insert("apps".into(), json!(screen.apps));
769                out.insert("count".into(), json!(screen.elements.len()));
770                out.insert("seq".into(), json!(seq));
771                out.insert("elements".into(), json!(screen.elements));
772            }
773        Ok(out)
774    }
775
776    /// The last known screen size, when `read_failed` is the phone's
777    /// accessibility daemon giving no reading on an unlocked phone. Control
778    /// Center does that while a screen recording runs, until it is closed
779    /// (2026-11-12, over Wi-Fi and USB alike), and closing it takes a tap, a
780    /// swipe or a button: input that needs no element then goes out on this
781    /// size, unverified.
782    fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
783        if !daemon_silent(read_failed) {
784            return None;
785        }
786        self.screen_size
787    }
788
789    /// A tap on a point while the screen cannot be read (`unread_size`).
790    /// Nothing is read afterwards either: that would only wait out another
791    /// timeout.
792    async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
793        let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
794            return Err(read_failed);
795        };
796        let (x, y) = (*x, *y);
797        if x < 0.0 || y < 0.0 || x > w || y > h {
798            return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
799        }
800        let (nx, ny) = (normalize(x, w), normalize(y, h));
801        let gesture = if args.double {
802            Gesture::DoubleTap { x: nx, y: ny }
803        } else {
804            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
805        };
806        self.send_gesture(gesture).await?;
807        Ok(fields! {
808            "sent": true,
809            "verified": false,
810            "screen_changed": false,
811            "x": (x * 10.0).round() / 10.0,
812            "y": (y * 10.0).round() / 10.0,
813            "note": UNREAD_NOTE,
814        })
815    }
816
817    /// Tap several targets in a row on the screen as it is now: every target
818    /// is found before the first tap goes
819    /// out, nothing is re-read in between, and the screen is read once at the
820    /// end. For keypads and other screens that hold still while tapped.
821    pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
822        if targets.is_empty() {
823            return Err(Fail::invalid("the tap list is empty"));
824        }
825        let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
826        let (screen, points) = self.locate_ready(&parsed).await?;
827        let before = screen.signature();
828        let normalized: Vec<(u16, u16)> = points
829            .iter()
830            .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
831            .collect();
832        let tapped: Vec<Value> = points
833            .iter()
834            .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
835            .collect();
836        let input_began = Instant::now();
837        for (i, (nx, ny)) in normalized.iter().enumerate() {
838            let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
839            if let Err(mut fail) = self.send_gesture(tap).await {
840                if i > 0 {
841                    // The earlier taps of the list did go out.
842                    fail.outcome = crate::reply::Outcome::Unknown;
843                }
844                return Err(fail
845                    .hint(t(
846                        "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
847                        "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
848                    ))
849                    .with("taps_sent", i));
850            }
851        }
852        let after = self.observe_with(input_began, AFTER_TAP, true).await;
853        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
854        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
855        let mut out = fields! {
856            "sent": true,
857            "verified": changed.is_some(),
858            "screen_changed": changed.unwrap_or(false),
859            "taps": tapped.len(),
860            "elements": tapped,
861        };
862        if let Some(change) = app_change {
863            out.insert("app_changed".into(), json!(true));
864            out.insert("app".into(), change);
865        }
866        Ok(out)
867    }
868
869    /// Drag between two points given in logical points and report whether the
870    /// screen changed.
871    pub(crate) async fn drag_points(
872        &mut self,
873        from: (f64, f64),
874        to: (f64, f64),
875        duration: Duration,
876        motion: Motion,
877    ) -> Res<(bool, Option<Screen>)> {
878        let (_, screen) = self
879            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
880            .await?;
881        if screen.width <= 0.0 || screen.height <= 0.0 {
882            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
883        }
884        for (x, y) in [from, to] {
885            if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
886                return Err(Fail::invalid(format!(
887                    "point {x},{y} is outside the {}x{} screen",
888                    screen.width, screen.height
889                )));
890            }
891        }
892        let before = screen.signature();
893        let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
894        let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
895        let input_began = Instant::now();
896        self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
897        let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
898        if motion == Motion::Flick {
899            // The content coasts after a flick: read until two readings agree.
900            for _ in 0..3 {
901                let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
902                let still = match (&after, &next) {
903                    (Some(a), Some(b)) => a.signature() == b.signature(),
904                    _ => true,
905                };
906                after = next.or(after);
907                if still {
908                    break;
909                }
910            }
911        }
912        let changed = after.as_ref().is_some_and(|s| s.signature() != before);
913        Ok((changed, after))
914    }
915
916    async fn screen_size(&mut self) -> Res<(f64, f64)> {
917        let (_, screen) = self
918            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
919            .await?;
920        Ok((screen.width, screen.height))
921    }
922
923    pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
924        let read = self
925            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
926            .await;
927        let (w, h) = match &read {
928            Ok((_, screen)) => (screen.width, screen.height),
929            Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
930        };
931        let (from, to) = match (&args.direction, args.from, args.to) {
932            (Some(dir), None, None) => {
933                // The finger moves in the named direction across the middle.
934                let (cx, cy) = (w / 2.0, h / 2.0);
935                match dir.as_str() {
936                    "up" => ((cx, h * 0.70), (cx, h * 0.30)),
937                    "down" => ((cx, h * 0.30), (cx, h * 0.70)),
938                    "left" => ((w * 0.80, cy), (w * 0.20, cy)),
939                    "right" => ((w * 0.20, cy), (w * 0.80, cy)),
940                    other => {
941                        return Err(Fail::invalid(format!(
942                            "unknown direction `{other}` (up, down, left, right)"
943                        )));
944                    }
945                }
946            }
947            (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
948            _ => return Err(Fail::invalid("give a direction, or both from and to")),
949        };
950        let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
951        let motion = if args.drag { Motion::Place } else { Motion::Flick };
952        if read.is_err() {
953            // `unread_size`: drag on the last known size, read nothing after.
954            for (x, y) in [from, to] {
955                if x < 0.0 || y < 0.0 || x > w || y > h {
956                    return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
957                }
958            }
959            let a = (normalize(from.0, w), normalize(from.1, h));
960            let b = (normalize(to.0, w), normalize(to.1, h));
961            self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
962            return Ok(fields! {
963                "sent": true,
964                "verified": false,
965                "screen_changed": false,
966                "from": [from.0, from.1],
967                "to": [to.0, to.1],
968                "note": UNREAD_NOTE,
969            });
970        }
971        let (changed, after) = self.drag_points(from, to, duration, motion).await?;
972        Ok(fields! {
973            "sent": true,
974            "verified": after.is_some(),
975            "screen_changed": changed,
976            "from": [from.0, from.1],
977            "to": [to.0, to.1],
978        })
979    }
980
981    /// Two fingers pinching, spreading or turning around a point. Arguments
982    /// that cannot make a gesture, or a point with no room for one, fail
983    /// before anything is sent. `screen_changed: false` is a fair answer on a
984    /// screen that does not zoom.
985    pub async fn cmd_pinch(&mut self, args: PinchArgs) -> Res {
986        let (scale, rotate) = args.motion()?;
987        let duration = Duration::from_millis(args.duration_ms.unwrap_or(500).clamp(100, 10_000));
988        let target = (!args.at.is_null()).then(|| parse_target(&args.at)).transpose()?;
989        let read = match &target {
990            Some(target) => self.locate_ready(std::slice::from_ref(target)).await.map(|(screen, mut points)| {
991                let (x, y, _) = points.remove(0);
992                (screen, Some((x, y)))
993            }),
994            None => self
995                .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
996                .await
997                .map(|(_, screen)| (screen, None)),
998        };
999        let (size, at, before) = match &read {
1000            Ok((screen, _)) if screen.width <= 0.0 || screen.height <= 0.0 => {
1001                return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
1002            }
1003            Ok((screen, at)) => ((screen.width, screen.height), *at, Some(screen.signature())),
1004            Err(read_failed) => {
1005                // `unread_size`: only the middle or a point can be pinched without a reading.
1006                let size = self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?;
1007                let at = match &target {
1008                    None => None,
1009                    Some(Target::Point(x, y)) => Some((*x, *y)),
1010                    Some(Target::Selector(_)) => return Err(read_failed.clone()),
1011                };
1012                (size, at, None)
1013            }
1014        };
1015        let center = at.unwrap_or((size.0 / 2.0, size.1 / 2.0));
1016        let plan = pinch::plan(&pinch::Spec { screen: size, center, scale, rotate, duration }).map_err(Fail::invalid)?;
1017        let span = [(plan.span.0 * 10.0).round() / 10.0, (plan.span.1 * 10.0).round() / 10.0];
1018        let input_began = Instant::now();
1019        self.send_gesture(Gesture::Pinch(plan)).await?;
1020        let mut out = fields! {
1021            "sent": true,
1022            "center": [(center.0 * 10.0).round() / 10.0, (center.1 * 10.0).round() / 10.0],
1023            "scale": scale,
1024            "rotate": rotate,
1025            "span": span,
1026        };
1027        let Some(before) = before else {
1028            out.insert("verified".into(), json!(false));
1029            out.insert("screen_changed".into(), json!(false));
1030            out.insert("note".into(), json!(UNREAD_NOTE));
1031            return Ok(out);
1032        };
1033        let after = self.observe(input_began, AFTER_DRAG).await.map(|(_, screen)| screen);
1034        out.insert("verified".into(), json!(after.is_some()));
1035        out.insert("screen_changed".into(), json!(after.is_some_and(|s| s.signature() != before)));
1036        Ok(out)
1037    }
1038
1039    /// Move the content one step. "down" shows what is further down.
1040    async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
1041        let (w, h) = self.screen_size().await?;
1042        let cx = w / 2.0;
1043        let (from, to) = match direction {
1044            "down" => ((cx, h * 0.68), (cx, h * 0.32)),
1045            "up" => ((cx, h * 0.32), (cx, h * 0.68)),
1046            other => {
1047                return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
1048            }
1049        };
1050        // A drag that slows to a stop moves the list by the finger's distance,
1051        // without a fling that would overshoot.
1052        self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
1053            .await
1054    }
1055
1056    pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
1057        if args.direction != "up" && args.direction != "down" {
1058            return Err(Fail::invalid("direction must be up or down"));
1059        }
1060        let Some(until) = &args.until else {
1061            let (changed, after) = self.scroll_step(&args.direction).await?;
1062            return Ok(fields! {
1063                "sent": true,
1064                "verified": after.is_some(),
1065                "screen_changed": changed,
1066                "scrolls": 1,
1067            });
1068        };
1069        let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
1070        let max = args.max.unwrap_or(10).clamp(1, 100);
1071        let full = sel.needs_identifier();
1072        let mut scrolls = 0u32;
1073        let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
1074        loop {
1075            let matches = dedupe_targets(sel.find(&screen));
1076            // Found means: on screen and tappable (not under a toolbar).
1077            if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
1078                if matches.len() > 1 {
1079                    return Err(Fail::not_sent(
1080                        "ambiguous_selector",
1081                        format!("{} elements match {}", matches.len(), sel.describe()),
1082                    )
1083                    .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
1084                    .with("scrolls", scrolls));
1085                }
1086                return Ok(fields! {
1087                    "found": true,
1088                    "scrolls": scrolls,
1089                    "screen_changed": scrolls > 0,
1090                    "element": brief(el),
1091                });
1092            }
1093            if scrolls >= max {
1094                return Err(Fail::new(
1095                    crate::reply::Outcome::Done,
1096                    "not_found",
1097                    format!("{} not found after {scrolls} scrolls", sel.describe()),
1098                )
1099                .with("found", false)
1100                .with("scrolls", scrolls)
1101                .with("screen_summary", screen.summary(12)));
1102            }
1103            let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
1104            scrolls += 1;
1105            if !changed {
1106                // One still screen can be a dropped drag; two in a row is the
1107                // end of the list.
1108                (changed, after) = self.scroll_step(&args.direction).await?;
1109                scrolls += 1;
1110            }
1111            if !changed {
1112                return Err(Fail::new(
1113                    crate::reply::Outcome::Done,
1114                    "not_found",
1115                    format!(
1116                        "{} not found; the list stopped moving after {scrolls} scrolls",
1117                        sel.describe()
1118                    ),
1119                )
1120                .with("found", false)
1121                .with("scrolls", scrolls)
1122                .with("screen_summary", screen.summary(12)));
1123            }
1124            screen = match (after, full) {
1125                (Some(s), false) => s,
1126                _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
1127            };
1128        }
1129    }
1130
1131    pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
1132        let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
1133        let (_, screen) = self
1134            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1135            .await?;
1136        let before = Some(screen.signature());
1137        let input_began = Instant::now();
1138        let input = self.ensure_input().await?;
1139        if let Err(e) = input.key(usage, &modifiers).await {
1140            self.drop_input();
1141            return Err(Fail::unknown("input_failed", format!("{e:#}")));
1142        }
1143        let changed = match before {
1144            Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
1145            None => None,
1146        };
1147        Ok(fields! {
1148            "sent": true,
1149            "verified": changed.is_some(),
1150            "screen_changed": changed.unwrap_or(false),
1151            "key": args.key,
1152        })
1153    }
1154
1155    pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
1156        let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
1157        // A button needs no screen size; the reading is only to compare with.
1158        let before = match self
1159            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1160            .await
1161        {
1162            Ok((_, screen)) => Some(screen.signature()),
1163            Err(read_failed) if daemon_silent(&read_failed) => None,
1164            Err(read_failed) => return Err(read_failed),
1165        };
1166        let input_began = Instant::now();
1167        self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
1168        let changed = match &before {
1169            Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
1170            None => None,
1171        };
1172        let mut out = fields! {
1173            "sent": true,
1174            "verified": changed.is_some(),
1175            "screen_changed": changed.unwrap_or(false),
1176            "button": args.button,
1177        };
1178        if before.is_none() {
1179            out.insert("note".into(), json!(UNREAD_NOTE));
1180        }
1181        Ok(out)
1182    }
1183
1184    pub async fn cmd_apps(&mut self) -> Res {
1185        let apps = self.list_apps(true).await?;
1186        let list: Vec<Value> = apps
1187            .iter()
1188            .map(|a| {
1189                json!({
1190                    "bundle_id": a.bundle_identifier,
1191                    "name": a.name,
1192                    "version": a.version,
1193                    "first_party": a.is_first_party,
1194                    "removable": a.is_removable,
1195                })
1196            })
1197            .collect();
1198        Ok(fields! { "count": list.len(), "apps": list })
1199    }
1200
1201    async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
1202        // Something with a dot and no spaces is a bundle id; anything else is
1203        // looked up by name.
1204        if app.contains('.') && !app.contains(' ') {
1205            return Ok(app.to_string());
1206        }
1207        let want = nfc(app).to_lowercase();
1208        let apps = self.list_apps(false).await?;
1209        let hits: Vec<&idevice::core_device::AppListEntry> = apps
1210            .iter()
1211            .filter(|a| nfc(&a.name).to_lowercase() == want)
1212            .collect();
1213        match hits.len() {
1214            1 => Ok(hits[0].bundle_identifier.clone()),
1215            0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
1216                .hint(t(
1217                    "`iph apps`로 이름과 bundle id를 확인하세요.",
1218                    "See names and bundle ids with `iph apps`.",
1219                ))),
1220            _ => Err(Fail::not_sent(
1221                "ambiguous_app",
1222                format!("{} apps are named \"{app}\"", hits.len()),
1223            )
1224            .with(
1225                "candidates",
1226                Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
1227            )),
1228        }
1229    }
1230
1231    pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
1232        args.validate()?;
1233        let bundle = self.resolve_bundle(&args.app).await?;
1234        let mut svc = self.app_service().await?;
1235        let arguments: Vec<&str> = args.arguments.iter().map(String::as_str).collect();
1236        let launch_began = Instant::now();
1237        // From here the launch request may have reached the phone.
1238        let launched = tokio::time::timeout(
1239            Duration::from_secs(30),
1240            svc.launch_application(
1241                bundle.as_str(),
1242                &arguments,
1243                args.kill_existing(),
1244                false,
1245                args.environment_dictionary(),
1246                None,
1247                None,
1248            ),
1249        )
1250        .await
1251        .map_err(|_| "no answer within 30 s".to_string())
1252        .and_then(|r| r.map_err(|e| format!("{e:?}")));
1253        let resp = match launched {
1254            Ok(r) => r,
1255            Err(msg) => {
1256                let locked = msg.to_lowercase().contains("lock");
1257                return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1258                    t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1259                } else {
1260                    t(
1261                        "bundle id가 맞는지 `iph apps`로 확인하세요.",
1262                        "Check the bundle id with `iph apps`.",
1263                    )
1264                }));
1265            }
1266        };
1267        let pid = resp.pid as i64;
1268        // The app is in front when a reading lists it as an application.
1269        let mut foreground = false;
1270        let mut delay = AFTER_LAUNCH;
1271        for _ in 0..4 {
1272            if let Some((_, screen)) = self.observe(launch_began, delay).await
1273                && screen.apps.iter().any(|a| a.pid == pid) {
1274                    foreground = true;
1275                    break;
1276                }
1277            delay = Duration::from_millis(300);
1278        }
1279        let mut out = fields! {
1280            "sent": true,
1281            "verified": foreground,
1282            "foreground": foreground,
1283            "bundle_id": bundle,
1284            "pid": pid,
1285            "restart": args.kill_existing(),
1286        };
1287        if args.has_launch_options() {
1288            out.insert("restart_implied".into(), json!(!args.restart));
1289            out.insert("arguments".into(), json!(args.arguments));
1290            out.insert("environment_keys".into(), json!(args.environment.keys().collect::<Vec<_>>()));
1291        }
1292        Ok(out)
1293    }
1294
1295    pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1296        let png = self.screenshot_png().await?;
1297        let path = std::path::PathBuf::from(&args.path);
1298        if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1299            std::fs::create_dir_all(dir)
1300                .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1301        }
1302        std::fs::write(&path, &png)
1303            .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1304        Ok(fields! { "path": args.path, "bytes": png.len() })
1305    }
1306
1307    /// The text element that has the keyboard, if exactly one does.
1308    fn focused_field(screen: &Screen) -> Option<&Element> {
1309        let mut focused = screen.elements.iter().filter(|e| e.focused);
1310        let first = focused.next()?;
1311        focused.next().is_none().then_some(first)
1312    }
1313
1314    /// What a field shows after a paste, asked of the phone directly: again
1315    /// every 0.1 s until it differs from before or
1316    /// `PASTE_SHOWS` has passed. `None` when the phone does not answer, and
1317    /// the caller reads the whole screen instead.
1318    async fn field_value(&mut self, field: &Element) -> Option<String> {
1319        let key = field.key?;
1320        let before = field.value.as_deref();
1321        let began = Instant::now();
1322        loop {
1323            let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1324            let now = crate::ax::tree::value_text(Some(&value))?;
1325            if Some(now.as_str()) != before || began.elapsed() >= PASTE_SHOWS {
1326                return Some(now);
1327            }
1328            tokio::time::sleep(Duration::from_millis(100)).await;
1329        }
1330    }
1331
1332    pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1333        if args.text.is_empty() && !args.clear {
1334            return Err(Fail::invalid("nothing to type"));
1335        }
1336        if args.text.len() > 1024 * 1024 {
1337            return Err(Fail::invalid("text is larger than 1 MiB"));
1338        }
1339        let (_, screen) = self
1340            .screen(Fresh::Force, false, BuildOptions::default())
1341            .await?;
1342        let Some(field) = Self::focused_field(&screen).cloned() else {
1343            return Err(Fail::not_sent(
1344                "no_focused_field",
1345                "no text field has the keyboard right now",
1346            )
1347            .hint(t(
1348                "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1349                "Tap the field first with `iph tap` so the keyboard comes up.",
1350            ))
1351            .with("screen_summary", screen.summary(12)));
1352        };
1353        trace(|| "type: focused field found".into());
1354
1355        // The input session first: if it cannot be had, nothing has been
1356        // touched yet, the clipboard included.
1357        self.ensure_input().await?;
1358        let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1359
1360        // Remember what the user had on the clipboard and, meanwhile, register
1361        // the temporary keyboard (no key goes out yet).
1362        let input = self.ensure_input().await?;
1363        let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1364        trace(|| "type: clipboard saved, keyboard attached".into());
1365        if let Err(e) = attached {
1366            // Nothing has been touched: the clipboard still holds the user's content.
1367            return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1368        }
1369        if !args.text.is_empty()
1370            && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1371                let _ = input.keyboard_detach().await;
1372                self.invalidate_readings();
1373                return Err(Fail::not_sent("clipboard_failed", e));
1374            }
1375        trace(|| "type: clipboard set".into());
1376
1377        let cmd = vec![crate::keys::MOD_COMMAND];
1378        let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1379        if args.clear {
1380            combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1381            combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1382        }
1383        if !args.text.is_empty() {
1384            combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1385        }
1386        let keys_began = Instant::now();
1387        let pressed = input.keyboard_press(&combos).await;
1388        trace(|| "type: keys sent".into());
1389        if let Err(e) = pressed {
1390            let _ = input.keyboard_detach().await;
1391            self.drop_input();
1392            self.invalidate_readings();
1393            let restored = saved.restore(&adapter, port, &args.text).await;
1394            return Err(Fail::unknown("input_failed", format!("{e:#}"))
1395                .hint(t(
1396                    "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1397                    "The input may have gone out. Do not resend; check the screen with `iph read`.",
1398                ))
1399                .with("clipboard_restored", restored));
1400        }
1401
1402        // The paste keys have gone out, so the clipboard can go back while
1403        // the field is read. The temporary keyboard stays attached meanwhile:
1404        // the screen holds still and Return can follow at once. What the field
1405        // shows is reported, not judged: apps format, mask, correct and clear
1406        // what was pasted (₩ 30,000 for 30000), so only the caller can tell
1407        // whether it is right, as with Playwright's fill().
1408        let restoring = saved.restore(&adapter, port, &args.text);
1409        let reading = async {
1410            tokio::time::sleep(AFTER_PASTE).await;
1411            if let Some(value) = self.field_value(&field).await {
1412                return Some(value);
1413            }
1414            let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1415            // Focus may have moved to a permission dialog or another field.
1416            // Only the original field can tell us what this paste changed.
1417            screen.by_id(&field.id).and_then(|e| e.value.clone())
1418        };
1419        let (restored, value) = tokio::join!(restoring, reading);
1420        trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1421        let observation = TypeObservation::new(&args, field.value.clone(), value);
1422
1423        let mut submitted = Some(false);
1424        let mut submit_error = None;
1425        if args.submit && observation.can_submit() {
1426            if let Some(input) = self.input.as_mut() {
1427                match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1428                    Ok(()) => submitted = Some(true),
1429                    Err(e) => {
1430                        submitted = None;
1431                        submit_error = Some(format!("{e:#}"));
1432                    }
1433                }
1434            } else {
1435                submit_error = Some("the input session closed before Return could be sent".into());
1436            }
1437        }
1438        if let Some(input) = self.input.as_mut()
1439            && input.keyboard_detach().await.is_err() {
1440                self.drop_input();
1441            }
1442        // The on-screen keyboard comes back now; readings from before show
1443        // the screen without it.
1444        self.invalidate_readings();
1445
1446        if submit_error.is_some() {
1447            self.drop_input();
1448        }
1449        // Compare complete readings after the temporary keyboard is gone.
1450        // A field read alone cannot tell whether the rest of the screen moved.
1451        let screen_changed = self
1452            .observe(keys_began, AFTER_PASTE)
1453            .await
1454            .map(|(_, after)| after.signature() != screen.signature());
1455        let mut result = observation.result(
1456            &args,
1457            &field,
1458            restored,
1459            submitted,
1460            screen_changed,
1461            submit_error,
1462        );
1463        if !restored && saved.unreadable() {
1464            let out = match &mut result {
1465                Ok(out) => out,
1466                Err(fail) => &mut fail.extra,
1467            };
1468            out.insert(
1469                "clipboard_note".into(),
1470                json!(t(
1471                    "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1472                    "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1473                )),
1474            );
1475        }
1476        result
1477    }
1478}
1479
1480const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1481
1482/// Kinds of data the phone's clipboard service holds itself: what `iph type`
1483/// puts there, and plain text or a picture. An item that carries any other kind
1484/// (rich text, an app's own format) was copied in an app, and its bytes are
1485/// only promised: the service asks that app for them when someone reads them.
1486/// Read from here, that request never came back. Text copied in Notes stalled
1487/// past 8 s for every kind on offer (rtf and plain text alike), and the
1488/// service then answered nothing at all, writes included, for about a minute
1489/// (2026-11-11: `iph type` right after ⌘C in Notes failed twice with "set
1490/// clipboard: no answer" and lost the clipboard the third time). Such content
1491/// is not read.
1492const SERVICE_KINDS: [&str; 6] = [
1493    "public.utf8-plain-text",
1494    "public.plain-text",
1495    "public.text",
1496    "public.png",
1497    "public.jpeg",
1498    "public.tiff",
1499];
1500/// What the service's own items took to read: 75 ~ 124 ms.
1501const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1502
1503/// Whether an item with these kinds was copied in an app, so reading its
1504/// bytes would wait on that app.
1505fn app_provided(types: &[String]) -> bool {
1506    types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1507}
1508
1509/// What the clipboard holds, as far as it can be read without waiting on an app.
1510#[derive(Debug, PartialEq)]
1511enum Peek {
1512    Empty,
1513    Text(String),
1514    Image(String, Vec<u8>),
1515    /// Copied in an app; left unread (see `SERVICE_KINDS`).
1516    AppContent,
1517    /// No answer, or something not understood.
1518    Unknown,
1519}
1520
1521/// What the user had on the phone's clipboard before `iph type` borrowed it.
1522struct Clipboard {
1523    before: Peek,
1524}
1525
1526impl Clipboard {
1527    /// Every call uses its own connection, so a late answer to one request
1528    /// cannot be taken for the answer to the next.
1529    async fn connect(
1530        adapter: &crate::tcp::handle::AdapterHandle,
1531        port: u16,
1532    ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1533        use idevice::RsdService;
1534        let mut adapter = adapter.clone();
1535        let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1536            .await
1537            .map_err(|_| "clipboard service: no answer".to_string())?
1538            .map_err(|e| format!("clipboard service: {e}"))?;
1539        idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1540            .await
1541            .map_err(|e| format!("clipboard service: {e:?}"))
1542    }
1543
1544    /// Look at the clipboard. Bytes come along only as promises, and only
1545    /// those the service holds itself are fetched.
1546    async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1547        use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1548        let general = idevice::core_device::GENERAL_PASTEBOARD;
1549        let Ok(mut clip) = Self::connect(adapter, port).await else {
1550            return Peek::Unknown;
1551        };
1552        let pulled = tokio::time::timeout(
1553            Duration::from_secs(8),
1554            clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1555        )
1556        .await;
1557        let Ok(Ok(snap)) = pulled else {
1558            return Peek::Unknown;
1559        };
1560        trace(|| {
1561            let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1562            format!("clipboard: kinds [{}]", kinds.join(" | "))
1563        });
1564        let Some(item) = snap.items.first() else {
1565            return Peek::Empty;
1566        };
1567        if snap.items.iter().any(|i| app_provided(&i.types)) {
1568            return Peek::AppContent;
1569        }
1570        // Text before a picture, in the order of `SERVICE_KINDS`.
1571        for kind in SERVICE_KINDS {
1572            let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1573                continue;
1574            };
1575            let bytes = match &entry.payload {
1576                PasteboardPayload::Inline(b) => Some(b.clone()),
1577                PasteboardPayload::Promised { .. } => {
1578                    match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1579                        Ok(Ok(b)) => b,
1580                        Ok(Err(_)) => None,
1581                        // Whatever holds it up may hold up the service too.
1582                        Err(_) => return Peek::Unknown,
1583                    }
1584                }
1585                PasteboardPayload::Error(_) => None,
1586            };
1587            let Some(bytes) = bytes else { continue };
1588            return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1589                Peek::Image(kind.to_string(), bytes)
1590            } else {
1591                match String::from_utf8(bytes) {
1592                    Ok(text) => Peek::Text(text),
1593                    Err(_) => continue,
1594                }
1595            };
1596        }
1597        Peek::Unknown
1598    }
1599
1600    async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1601        let mut clip = Self::connect(adapter, port).await?;
1602        let general = idevice::core_device::GENERAL_PASTEBOARD;
1603        match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1604            Ok(Ok(())) => Ok(()),
1605            Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1606            Err(_) => Err("set clipboard: no answer".into()),
1607        }
1608    }
1609
1610    async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1611        Clipboard { before: Self::peek(adapter, port).await }
1612    }
1613
1614    /// True when what was on the clipboard could not be read and is lost
1615    /// once `iph type` puts its text there.
1616    fn unreadable(&self) -> bool {
1617        matches!(self.before, Peek::AppContent | Peek::Unknown)
1618    }
1619
1620    /// Put the old content back. True when the clipboard is known to hold
1621    /// what it held before.
1622    async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1623        if typed.is_empty() {
1624            return true;
1625        }
1626        // Give the paste a moment to read the clipboard.
1627        tokio::time::sleep(Duration::from_millis(150)).await;
1628        // Only touch the clipboard if it still holds what we put there; the
1629        // user may have copied something else meanwhile.
1630        if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1631            return false;
1632        }
1633        let general = idevice::core_device::GENERAL_PASTEBOARD;
1634        let wrote = match &self.before {
1635            Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1636            Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1637                Ok(mut clip) => {
1638                    matches!(
1639                        tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1640                        Ok(Ok(()))
1641                    )
1642                }
1643                Err(_) => false,
1644            },
1645            // The clipboard was empty, or what was there is gone: either way
1646            // the typed text does not stay behind.
1647            Peek::Empty | Peek::AppContent | Peek::Unknown => {
1648                let _ = Self::set_text(adapter, port, "").await;
1649                return matches!(self.before, Peek::Empty);
1650            }
1651        };
1652        if !wrote {
1653            return false;
1654        }
1655        match (Self::peek(adapter, port).await, &self.before) {
1656            (Peek::Text(now), Peek::Text(old)) => &now == old,
1657            (Peek::Image(..), Peek::Image(..)) => true,
1658            _ => false,
1659        }
1660    }
1661}
1662
1663#[cfg(test)]
1664mod type_tests {
1665    use super::*;
1666    use crate::reply::envelope;
1667
1668    fn field(before: Option<&str>) -> Element {
1669        serde_json::from_value(json!({
1670            "id": "original", "role": "text_field", "value": before, "focused": true,
1671        }))
1672        .unwrap()
1673    }
1674
1675    fn result(
1676        args: &TypeArgs,
1677        before: Option<&str>,
1678        after: Option<&str>,
1679        screen: Option<bool>,
1680    ) -> Value {
1681        let field = field(before);
1682        let observed = TypeObservation::new(args, field.value.clone(), after.map(String::from));
1683        let submit = args.submit && observed.can_submit();
1684        envelope(
1685            observed.result(args, &field, true, Some(submit), screen, None),
1686            Instant::now(),
1687        )
1688    }
1689
1690    #[test]
1691    fn changed_values_are_observations_not_exact_text_matches() {
1692        for value in ["30000", "₩ 30,000", "•••••", "corrected text", " \n "] {
1693            let args = TypeArgs {
1694                text: "30000".into(),
1695                submit: true,
1696                ..Default::default()
1697            };
1698            let out = result(&args, Some(""), Some(value), Some(true));
1699            assert_eq!(out["ok"], true, "{out}");
1700            assert_eq!(out["outcome"], "done");
1701            assert_eq!(out["verified"], true);
1702            assert_eq!(out["value_before"], "");
1703            assert_eq!(out["value"], value);
1704            assert_eq!(out["value_changed"], true);
1705            assert_eq!(out["screen_changed"], true);
1706            assert_eq!(out["submit_sent"], true);
1707            assert!(out.get("warning").is_none());
1708        }
1709    }
1710
1711    #[test]
1712    fn unchanged_paste_warns_without_claiming_permission_was_denied() {
1713        let args = TypeArgs {
1714            text: "테스트 문장".into(),
1715            ..Default::default()
1716        };
1717        for value in ["", "old text", "•••••"] {
1718            let out = result(&args, Some(value), Some(value), Some(false));
1719            assert_eq!(out["ok"], true);
1720            assert_eq!(out["outcome"], "done");
1721            assert_eq!(out["sent"], true);
1722            assert_eq!(out["verified"], true, "verified still means read back");
1723            assert_eq!(out["value_changed"], false);
1724            assert_eq!(out["screen_changed"], false);
1725            assert_eq!(out["warning"]["code"], "value_unchanged");
1726            assert!(
1727                out["warning"]["message"]
1728                    .as_str()
1729                    .unwrap()
1730                    .contains("may not")
1731            );
1732            assert!(out["warning"]["hint"].as_str().is_some());
1733        }
1734        // A dialog or another part of the screen may change while the field does not.
1735        let out = result(&args, Some(""), Some(""), Some(true));
1736        assert_eq!(out["value_changed"], false);
1737        assert_eq!(out["screen_changed"], true);
1738        assert!(out.get("warning").is_some());
1739    }
1740
1741    #[test]
1742    fn unreadable_and_uncomparable_values_do_not_become_empty_strings() {
1743        let args = TypeArgs {
1744            text: "hello".into(),
1745            ..Default::default()
1746        };
1747        let out = result(&args, Some("old"), None, None);
1748        assert_eq!(out["ok"], true);
1749        assert_eq!(out["verified"], false);
1750        assert!(out["value"].is_null());
1751        assert!(out["value_changed"].is_null());
1752        assert!(out["screen_changed"].is_null());
1753        assert_eq!(out["warning"]["code"], "value_unreadable");
1754        let out = result(&args, None, Some("hello"), Some(true));
1755        assert_eq!(out["verified"], true);
1756        assert!(out["value_before"].is_null());
1757        assert!(out["value_changed"].is_null());
1758        assert_eq!(out["warning"]["code"], "value_change_unknown");
1759        // A readable field does not imply a complete screen comparison succeeded.
1760        let out = result(&args, Some(""), Some("hello"), None);
1761        assert_eq!(out["verified"], true);
1762        assert!(out["screen_changed"].is_null());
1763    }
1764
1765    #[test]
1766    fn submit_is_withheld_when_the_input_effect_needs_review() {
1767        let args = TypeArgs {
1768            text: "hello".into(),
1769            submit: true,
1770            ..Default::default()
1771        };
1772        for (before, after, code) in [
1773            (Some(""), Some(""), "value_unchanged"),
1774            (Some("old"), Some("old"), "value_unchanged"),
1775            (Some("old"), None, "value_unreadable"),
1776            (None, Some("hello"), "value_change_unknown"),
1777            (Some("old"), Some(""), "value_empty"),
1778        ] {
1779            let out = result(&args, before, after, Some(false));
1780            assert_eq!(out["ok"], false, "{out}");
1781            assert_eq!(out["outcome"], "unknown");
1782            assert_eq!(out["sent"], true);
1783            assert_eq!(out["submit_sent"], false);
1784            assert_eq!(out["error"]["code"], "type_not_verified");
1785            assert_eq!(out["warning"]["code"], code);
1786            assert_eq!(out["value"], json!(after));
1787            assert_eq!(out["clipboard_restored"], true);
1788        }
1789    }
1790
1791    #[test]
1792    fn clear_and_identical_replacement_are_reported_separately() {
1793        let mut args = TypeArgs {
1794            clear: true,
1795            submit: true,
1796            ..Default::default()
1797        };
1798        for before in [Some(""), Some("old"), None] {
1799            let out = result(&args, before, Some(""), Some(false));
1800            assert_eq!(out["ok"], true);
1801            assert_eq!(out["submit_sent"], true);
1802            assert!(out.get("warning").is_none());
1803        }
1804        let out = result(&args, Some("old"), Some("old"), Some(false));
1805        assert_eq!(out["warning"]["code"], "clear_not_observed");
1806        assert_eq!(out["submit_sent"], false);
1807        // Replacing with identical text can be intentional, but it does not
1808        // establish whether paste worked. The caller must inspect before Return.
1809        args.text = "same".into();
1810        let out = result(&args, Some("same"), Some("same"), Some(false));
1811        assert_eq!(out["outcome"], "unknown");
1812        assert_eq!(out["warning"]["code"], "value_unchanged");
1813    }
1814
1815    #[test]
1816    fn failed_return_keeps_observations_and_does_not_claim_it_was_unsent() {
1817        let args = TypeArgs {
1818            text: "hello".into(),
1819            submit: true,
1820            ..Default::default()
1821        };
1822        let field = field(Some(""));
1823        let observed = TypeObservation::new(&args, field.value.clone(), Some("hello".into()));
1824        let out = envelope(
1825            observed.result(
1826                &args,
1827                &field,
1828                false,
1829                None,
1830                None,
1831                Some("write failed".into()),
1832            ),
1833            Instant::now(),
1834        );
1835        assert_eq!(out["outcome"], "unknown");
1836        assert_eq!(out["error"]["code"], "input_failed");
1837        assert!(out["submit_sent"].is_null());
1838        assert_eq!(out["value_changed"], true);
1839        assert_eq!(out["clipboard_restored"], false);
1840    }
1841}
1842
1843#[cfg(test)]
1844mod clipboard_tests {
1845    use super::app_provided;
1846
1847    fn kinds(k: &[&str]) -> Vec<String> {
1848        k.iter().map(|s| s.to_string()).collect()
1849    }
1850
1851    #[test]
1852    fn notes_copy_is_app_content() {
1853        // What ⌘C in Notes put on the clipboard (2026-11-11).
1854        assert!(app_provided(&kinds(&[
1855            "public.utf8-plain-text",
1856            "com.apple.notes.richtext",
1857            "com.apple.webarchive",
1858            "com.apple.flat-rtfd",
1859            "iOS rich content paste pasteboard type",
1860            "public.html",
1861            "net.daringfireball.markdown",
1862            "public.rtf",
1863        ])));
1864    }
1865
1866    #[test]
1867    fn what_iph_puts_there_is_not() {
1868        // `set_text` writes these three kinds.
1869        assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1870        assert!(!app_provided(&kinds(&["public.png"])));
1871    }
1872}
1873
1874#[cfg(test)]
1875mod launch_tests {
1876    use super::LaunchArgs;
1877    use crate::actions::parse_args;
1878    use serde_json::json;
1879
1880    fn parse(v: serde_json::Value) -> Result<LaunchArgs, crate::reply::Fail> {
1881        parse_args::<LaunchArgs>(&v)
1882    }
1883
1884    #[test]
1885    fn a_plain_launch_sends_nothing_extra() {
1886        let a = parse(json!({"app": "x"})).unwrap();
1887        assert!(!a.kill_existing());
1888        assert!(a.environment_dictionary().is_none());
1889    }
1890
1891    #[test]
1892    fn arguments_or_environment_imply_a_restart() {
1893        let a = parse(json!({"app": "x", "arguments": ["-k", "v"]})).unwrap();
1894        assert!(a.kill_existing() && !a.restart);
1895        let b = parse(json!({"app": "x", "environment": {"TZ": "UTC"}})).unwrap();
1896        assert!(b.kill_existing());
1897        let dict = b.environment_dictionary().unwrap();
1898        assert_eq!(dict.get("TZ").and_then(|v| v.as_string()), Some("UTC"));
1899    }
1900
1901    #[test]
1902    fn environment_values_may_hold_an_equals_sign() {
1903        let a = parse(json!({"app": "x", "environment": {"K": "a=b"}})).unwrap();
1904        assert!(a.validate().is_ok());
1905    }
1906
1907    #[test]
1908    fn environment_names_are_checked() {
1909        for name in ["", "A=B"] {
1910            let a = parse(json!({"app": "x", "environment": {name: "v"}})).unwrap();
1911            assert_eq!(a.validate().unwrap_err().code, "invalid_argument");
1912        }
1913    }
1914
1915    #[test]
1916    fn a_non_string_value_is_rejected_at_the_boundary() {
1917        assert_eq!(parse(json!({"app": "x", "environment": {"A": 1}})).unwrap_err().code, "invalid_argument");
1918        assert_eq!(parse(json!({"app": "x", "environment": ["A=1"]})).unwrap_err().code, "invalid_argument");
1919        assert_eq!(parse(json!({"app": "x", "arguments": "-a"})).unwrap_err().code, "invalid_argument");
1920        assert_eq!(parse(json!({"app": "x", "arguments": [1]})).unwrap_err().code, "invalid_argument");
1921    }
1922}
1923
1924#[cfg(test)]
1925mod pinch_tests {
1926    use super::*;
1927    use crate::{config::Config, reply::Outcome};
1928
1929    fn args(v: Value) -> PinchArgs {
1930        serde_json::from_value(v).unwrap()
1931    }
1932
1933    #[test]
1934    fn a_direction_means_a_scale() {
1935        assert_eq!(args(json!({"direction": "in"})).motion().unwrap(), (0.5, 0.0));
1936        assert_eq!(args(json!({"direction": "out"})).motion().unwrap(), (2.0, 0.0));
1937        assert_eq!(args(json!({"direction": "out", "rotate": -30})).motion().unwrap(), (2.0, -30.0));
1938        assert_eq!(args(json!({"scale": 1.5, "rotate": 90})).motion().unwrap(), (1.5, 90.0));
1939        assert_eq!(args(json!({"rotate": 45})).motion().unwrap(), (1.0, 45.0));
1940    }
1941
1942    #[test]
1943    fn arguments_that_cannot_make_a_gesture_are_invalid() {
1944        for bad in [
1945            json!({}),
1946            json!({"scale": 1}),
1947            json!({"direction": "out", "scale": 2}),
1948            json!({"direction": "sideways"}),
1949            json!({"scale": 0.2}),
1950            json!({"scale": 5}),
1951            json!({"rotate": 181}),
1952            json!({"direction": "in", "rotate": -181}),
1953        ] {
1954            let fail = args(bad.clone()).motion().unwrap_err();
1955            assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1956        }
1957        assert!(serde_json::from_value::<PinchArgs>(json!({"scale": 2, "zoom": 1})).is_err());
1958    }
1959
1960    #[tokio::test]
1961    async fn a_bad_request_fails_before_the_phone_is_asked_for_anything() {
1962        let mut s = Session::new("00000000-TESTONLY-NOPHONE".into(), Config::default());
1963        for bad in [json!({}), json!({"direction": "in", "scale": 2}), json!({"scale": 9})] {
1964            let fail = s.cmd_pinch(args(bad.clone())).await.unwrap_err();
1965            assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1966        }
1967        let fail = s.cmd_pinch(args(json!({"direction": "in", "at": 7}))).await.unwrap_err();
1968        assert_eq!(fail.outcome, Outcome::NotSent);
1969    }
1970}