Skip to main content

iphone_cli/
actions.rs

1//! The commands themselves, shared by the CLI, the MCP server, `batch` and
2//! scripts. Each takes JSON arguments and returns the fields of its result.
3
4use std::time::{Duration, Instant};
5
6use serde::Deserialize;
7use serde_json::{Value, json};
8use unicode_normalization::UnicodeNormalization;
9
10use crate::{
11    ax::{
12        attrs,
13        dump::ObjKey,
14        tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
15    },
16    device::{
17        ax::trace,
18        input::{Motion, normalize},
19        session::{Fresh, Gesture, Session, reuse},
20    },
21    fields,
22    i18n::t,
23    reply::{Fail, Res},
24    selector::{Selector, dedupe_targets},
25};
26
27/// How long after an input to ask for the new screen. Measured on iOS 27: a
28/// request made while a navigation animation still runs comes back about
29/// 0.4 s later than one made just after it (2026-11-10, 14 taps).
30pub const AFTER_TAP: Duration = Duration::from_millis(480);
31/// A drag that ends with the finger at rest only leaves a short bounce.
32pub const AFTER_DRAG: Duration = Duration::from_millis(300);
33pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
34pub const AFTER_PASTE: Duration = Duration::from_millis(200);
35/// How long a tap waits for the screen to start changing. A picker that
36/// another part of the app puts up came about 1.5 s after the tap
37/// (2026-11-12, 당근 '이미지 추가').
38pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
39/// How long after the tap a changed screen may take to hold still.
40pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
41/// A reading up to this old is used for a tap once the phone confirms, by a
42/// few quick questions, that the screen is still as read. Older ones are read
43/// again in full.
44const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
45/// How long a pasted text may take to show in its field.
46const PASTE_SHOWS: Duration = Duration::from_millis(1500);
47/// How long to wait for the answer to one quick question.
48const ASK_WAIT: Duration = Duration::from_millis(1000);
49/// An unasked dump this recently before an input means the app is one that
50/// triggers them; its readings are then double-checked.
51const UNASKED_MEMORY: Duration = Duration::from_secs(5);
52/// The result's note for an input sent while the screen could not be read.
53const UNREAD_NOTE: &str =
54    "the screen could not be read; sent on the last known screen size, not verified";
55
56#[derive(Debug, Default, Deserialize)]
57#[serde(default, deny_unknown_fields)]
58pub struct ReadArgs {
59    /// Only the foreground application's elements.
60    pub app: bool,
61    /// Include elements outside the screen.
62    pub all: bool,
63    /// Wait for the second pass so identifiers are present.
64    pub full: bool,
65    /// Attach every numbered attribute.
66    pub raw: bool,
67    /// Accept a cached reading up to this old instead of asking the phone.
68    pub max_age_ms: Option<u64>,
69}
70
71#[derive(Debug, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct FindArgs {
74    pub selector: Value,
75    #[serde(default)]
76    pub all: bool,
77}
78
79#[derive(Debug, Default, Deserialize)]
80#[serde(default, deny_unknown_fields)]
81pub struct TapArgs {
82    /// A selector (string or object), `"x,y"`, or `{x, y}`.
83    pub target: Value,
84    pub long_ms: Option<u64>,
85    pub double: bool,
86    /// Read the screen once right after the tap instead of waiting for it to
87    /// change and hold still.
88    pub no_wait: bool,
89}
90
91#[derive(Debug, Default, Deserialize)]
92#[serde(default, deny_unknown_fields)]
93pub struct SwipeArgs {
94    pub direction: Option<String>,
95    pub from: Option<[f64; 2]>,
96    pub to: Option<[f64; 2]>,
97    pub duration_ms: Option<u64>,
98    /// Slow to a stop before lifting (a drag) instead of flicking.
99    pub drag: bool,
100}
101
102#[derive(Debug, Default, Deserialize)]
103#[serde(default, deny_unknown_fields)]
104pub struct ScrollArgs {
105    pub direction: String,
106    pub until: Option<Value>,
107    pub max: Option<u32>,
108}
109
110#[derive(Debug, Default, Deserialize)]
111#[serde(default, deny_unknown_fields)]
112pub struct TypeArgs {
113    pub text: String,
114    pub clear: bool,
115    pub submit: bool,
116}
117
118/// Observations before Return. Reading a value is distinct from seeing it
119/// change, and neither proves that it equals the caller's intended text.
120struct TypeObservation {
121    before: Option<String>,
122    value: Option<String>,
123    changed: Option<bool>,
124    warning: Option<Value>,
125}
126
127impl TypeObservation {
128    fn new(args: &TypeArgs, before: Option<String>, value: Option<String>) -> Self {
129        let changed = before.as_ref().zip(value.as_ref()).map(|(a, b)| a != b);
130        let retry_hint = t(
131            "입력이 반영되었을 수 있습니다. 자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요.",
132            "Input may have taken effect. Do not automatically retype; check with `iph read` or a screenshot.",
133        );
134        let warning = if value.is_none() {
135            Some(json!({
136                "code": "value_unreadable",
137                "message": "the input was sent but the field's value could not be read afterwards",
138                "hint": retry_hint,
139            }))
140        } else if args.text.is_empty() && value.as_deref() == Some("") {
141            // Clearing an already empty field is an observable no-op.
142            None
143        } else if args.text.is_empty() {
144            Some(json!({
145                "code": "clear_not_observed",
146                "message": "the clear input was sent but the field still shows a nonempty value",
147                "hint": retry_hint,
148            }))
149        } else if changed == Some(false) {
150            Some(json!({
151                "code": "value_unchanged",
152                "message": "the field's value is unchanged; the paste may not have taken effect (identical replacement, masking or app behavior can also cause this)",
153                "hint": t(
154                    "자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요. 붙여넣기가 막혔을 수도 있습니다: 설정 → 앱 → 해당 앱 → 다른 앱에서 붙여넣기 → 허용. 값이 같다는 사실만으로 권한 거부를 확정할 수는 없습니다.",
155                    "Do not automatically retype; check with `iph read` or a screenshot. Pasting may be blocked: Settings → Apps → this app → Paste from Other Apps → Allow. An unchanged value alone does not prove permission was denied.",
156                ),
157            }))
158        } else if value.as_deref() == Some("") {
159            Some(json!({
160                "code": "value_empty",
161                "message": "the input was sent but the field shows an empty value; the app may have cleared it",
162                "hint": retry_hint,
163            }))
164        } else if changed.is_none() {
165            Some(json!({
166                "code": "value_change_unknown",
167                "message": "the field's value was read afterwards, but its previous value was unavailable for comparison",
168                "hint": retry_hint,
169            }))
170        } else {
171            None
172        };
173        Self {
174            before,
175            value,
176            changed,
177            warning,
178        }
179    }
180
181    fn can_submit(&self) -> bool {
182        self.warning.is_none()
183    }
184
185    fn result(
186        self,
187        args: &TypeArgs,
188        field: &Element,
189        restored: bool,
190        submitted: Option<bool>,
191        screen_changed: Option<bool>,
192        submit_error: Option<String>,
193    ) -> Res {
194        let blocked_submit = args.submit && !self.can_submit();
195        let mut out = fields! {
196            "sent": true,
197            "verified": self.value.is_some(),
198            "value_before": self.before,
199            "value": self.value,
200            "value_changed": self.changed,
201            "field": brief(field),
202            "submit_sent": submitted,
203            "clipboard_restored": restored,
204            "screen_changed": screen_changed,
205        };
206        if let Some(warning) = self.warning {
207            out.insert("warning".into(), warning);
208        }
209        let fail = if let Some(e) = submit_error {
210            Some(Fail::unknown("input_failed", e))
211        } else if blocked_submit {
212            Some(Fail::unknown(
213                "type_not_verified",
214                "the input was sent, but its effect is uncertain; Return was not sent",
215            ))
216        } else {
217            None
218        };
219        if let Some(mut fail) = fail {
220            fail.hint = Some(t(
221                "입력은 이미 나갔습니다. 자동으로 재입력하거나 제출하지 말고 `iph read`나 스크린샷으로 먼저 확인하세요.",
222                "Input has already been sent. Do not automatically retype or submit; check with `iph read` or a screenshot first.",
223            ).into());
224            fail.extra = out;
225            Err(fail)
226        } else {
227            Ok(out)
228        }
229    }
230}
231
232#[derive(Debug, Deserialize)]
233#[serde(deny_unknown_fields)]
234pub struct KeyArgs {
235    pub key: String,
236}
237
238#[derive(Debug, Deserialize)]
239#[serde(deny_unknown_fields)]
240pub struct ButtonArgs {
241    pub button: String,
242}
243
244#[derive(Debug, Default, Deserialize)]
245#[serde(deny_unknown_fields)]
246pub struct LaunchArgs {
247    pub app: String,
248    /// Quit the app first, so it starts from its first screen.
249    #[serde(default)]
250    pub restart: bool,
251}
252
253#[derive(Debug, Deserialize)]
254#[serde(deny_unknown_fields)]
255pub struct ScreenshotArgs {
256    pub path: String,
257}
258
259pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
260    let v = if args.is_null() { json!({}) } else { args.clone() };
261    serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
262}
263
264pub fn brief(el: &Element) -> Value {
265    let mut v = json!({ "id": el.id, "role": el.role });
266    if let Some(l) = &el.label {
267        v["label"] = json!(l);
268    }
269    if let Some(val) = &el.value {
270        v["value"] = json!(val);
271    }
272    if let Some(i) = &el.identifier {
273        v["identifier"] = json!(i);
274    }
275    if let Some(f) = &el.frame {
276        v["frame"] = json!(f);
277    }
278    v
279}
280
281pub enum Target {
282    Selector(Selector),
283    Point(f64, f64),
284}
285
286/// The reading failed because the phone's accessibility daemon gave none on a
287/// phone not known to be locked (`Session::unread_size`).
288fn daemon_silent(read_failed: &Fail) -> bool {
289    let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
290    read_failed.code == "ax_timeout" && !locked
291}
292
293pub fn parse_target(v: &Value) -> Res<Target> {
294    if let Some(obj) = v.as_object()
295        && let (Some(x), Some(y)) = (
296            obj.get("x").and_then(|n| n.as_f64()),
297            obj.get("y").and_then(|n| n.as_f64()),
298        )
299            && obj.len() == 2 {
300                return Ok(Target::Point(x, y));
301            }
302    if let Some(s) = v.as_str() {
303        let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
304        if parts.len() == 2
305            && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
306                return Ok(Target::Point(x, y));
307            }
308    }
309    Selector::from_value(v)
310        .map(Target::Selector)
311        .map_err(|e| Fail::invalid(e.to_string()))
312}
313
314/// The single element a selector names, or an error that lists what matched.
315pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
316    let matches = dedupe_targets(sel.find(screen));
317    match matches.len() {
318        1 => Ok(matches[0]),
319        0 => Err(Fail::not_sent(
320            "not_found",
321            format!("no element matches {}", sel.describe()),
322        )
323        .hint(t(
324            "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
325            "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
326        ))
327        .with("selector", sel.describe())
328        .with("screen_summary", screen.summary(12))),
329        n => Err(Fail::not_sent(
330            "ambiguous_selector",
331            format!("{n} elements match {}", sel.describe()),
332        )
333        .hint(t(
334            "role=, within=, index= 으로 하나만 남게 좁히세요.",
335            "Narrow it to one with role=, within= or index=.",
336        ))
337        .with("selector", sel.describe())
338        .with(
339            "candidates",
340            Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
341        )),
342    }
343}
344
345/// A point inside the visible part of `el` that no other element covers.
346fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
347    let frame = el.frame.ok_or_else(|| {
348        Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
349    })?;
350    let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
351    let visible = frame.intersection(&bounds).ok_or_else(|| {
352        Fail::not_sent("off_screen", "the element is outside the screen")
353            .hint(t(
354                "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
355                "Bring it on screen with `iph scroll down --until …`.",
356            ))
357            .with("element", brief(el))
358    })?;
359    let covered = |x: f64, y: f64| -> Option<&Element> {
360        let hit = screen.element_at(x, y)?;
361        if hit.id == el.id {
362            return None;
363        }
364        // A child of the target (a label inside a row) is fine; an element
365        // that merely overlaps it (a toolbar over a list) is not.
366        let inside = hit.frame.is_some_and(|h| {
367            h.x >= frame.x - 1.0
368                && h.y >= frame.y - 1.0
369                && h.x + h.w <= frame.x + frame.w + 1.0
370                && h.y + h.h <= frame.y + frame.h + 1.0
371        });
372        let encloses = hit.frame.is_some_and(|h| {
373            frame.x >= h.x - 1.0
374                && frame.y >= h.y - 1.0
375                && frame.x + frame.w <= h.x + h.w + 1.0
376                && frame.y + frame.h <= h.y + h.h + 1.0
377        });
378        (!inside && !encloses).then_some(hit)
379    };
380    let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
381    let mut blocker = None;
382    for (fx, fy) in candidates {
383        let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
384        match covered(x, y) {
385            None => return Ok((x, y)),
386            Some(b) => blocker = blocker.or(Some(b)),
387        }
388    }
389    Err(Fail::not_sent("covered", "another element covers the target")
390        .hint(t(
391            "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
392            "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
393        ))
394        .with("element", brief(el))
395        .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
396}
397
398fn nfc(s: &str) -> String {
399    s.nfc().collect()
400}
401
402/// The app now in front, when it is not the one that was in front before.
403fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
404    let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
405    let now = after.apps.first()?;
406    (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
407}
408
409impl Session {
410    fn build_opts(all: bool, raw: bool) -> BuildOptions {
411        BuildOptions { include_offscreen: all, raw }
412    }
413
414    pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
415        let fresh = match args.max_age_ms {
416            Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
417            None => Fresh::Force,
418        };
419        let (seq, mut screen) = self
420            .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
421            .await?;
422        if args.app
423            && let Some(front) = screen.apps.first().map(|a| a.pid) {
424                screen.elements.retain(|e| e.pid == front);
425                screen.apps.retain(|a| a.pid == front);
426            }
427        let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
428        Ok(fields! {
429            "screen": {"width": screen.width, "height": screen.height},
430            "apps": screen.apps,
431            "count": screen.elements.len(),
432            "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
433            "seq": seq,
434            "elements": screen.elements,
435        })
436    }
437
438    pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
439        let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
440        let (_, screen) = self
441            .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
442            .await?;
443        let matches = sel.find(&screen);
444        let mut out = fields! {
445            "selector": sel.describe(),
446            "count": matches.len(),
447            "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
448        };
449        if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
450            out.insert(
451                "note".into(),
452                json!("this reading came without identifiers, so id= could not match; try again"),
453            );
454        }
455        Ok(out)
456    }
457
458    /// Read the screen after an input that began at `since`: wait `delay` for
459    /// the screen to react, then ask the phone for a new dump.
460    ///
461    /// Some apps make the phone send dumps unasked while they change
462    /// (Calculator on every key). Such a dump shows the screen as it was a
463    /// moment earlier and can arrive late enough to pass for the answer; the
464    /// calculator example then read "12×3" instead of "408". So with `careful`
465    /// (inputs that change the screen several times), or when the phone has
466    /// lately sent unasked dumps, the screen is read until two readings in a
467    /// row agree.
468    pub(crate) async fn observe_with(
469        &mut self,
470        since: Instant,
471        delay: Duration,
472        careful: bool,
473    ) -> Option<(u64, Screen)> {
474        tokio::time::sleep(delay).await;
475        let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
476        let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
477        let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
478        if !careful && !chatty {
479            return Some(latest);
480        }
481        for _ in 0..2 {
482            trace(|| "observe: reading again until two readings agree".into());
483            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
484                break;
485            };
486            let same = next.1.signature() == latest.1.signature();
487            latest = next;
488            if same {
489                break;
490            }
491        }
492        Some(latest)
493    }
494
495    pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
496        self.observe_with(since, delay, false).await
497    }
498
499    /// Read the screen after an input until it has changed from `before` and
500    /// then holds still, as a browser waits for a page to load. The phone
501    /// sends no event when a screen comes up, so "came up" is two readings in
502    /// a row that agree. A screen that does not change within `CHANGE_WAIT`,
503    /// or keeps moving until `SETTLE_WAIT`, comes back as it is then.
504    pub(crate) async fn observe_settled(
505        &mut self,
506        since: Instant,
507        before: &[ElementSignature],
508    ) -> Option<(u64, Screen)> {
509        let mut latest = self.observe(since, AFTER_TAP).await?;
510        while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
511            trace(|| "observe: the screen has not changed yet; reading again".into());
512            match self.screen(Fresh::Force, false, BuildOptions::default()).await {
513                Ok(next) => latest = next,
514                Err(_) => return Some(latest),
515            }
516        }
517        if latest.1.signature() == before {
518            return Some(latest);
519        }
520        while since.elapsed() < SETTLE_WAIT {
521            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
522                break;
523            };
524            let same = next.1.signature() == latest.1.signature();
525            latest = next;
526            if same {
527                break;
528            }
529            trace(|| "observe: the screen is still moving; reading again".into());
530        }
531        Some(latest)
532    }
533
534    /// Read the screen and find where to tap for each target. Fails without
535    /// sending anything when a target is missing, ambiguous or covered.
536    async fn locate(
537        &mut self,
538        targets: &[Target],
539        fresh: Fresh,
540    ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
541        let need_ids = targets
542            .iter()
543            .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
544        let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
545        if screen.width <= 0.0 || screen.height <= 0.0 {
546            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
547        }
548        let mut points = Vec::with_capacity(targets.len());
549        for target in targets {
550            points.push(match target {
551                Target::Selector(sel) => {
552                    let el = resolve_one(sel, &screen)?;
553                    let (x, y) = tap_point(el, &screen)?;
554                    (x, y, Some(el.clone()))
555                }
556                Target::Point(x, y) => {
557                    if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
558                        return Err(Fail::invalid(format!(
559                            "point {x},{y} is outside the {}x{} screen",
560                            screen.width, screen.height
561                        )));
562                    }
563                    (*x, *y, screen.element_at(*x, *y).cloned())
564                }
565            });
566        }
567        Ok((screen, points))
568    }
569
570    /// Locate the targets on the latest reading when it is too old to reuse
571    /// as it is (an agent's turn takes seconds) but the phone confirms it:
572    /// every application still shows the same visible children (nothing came
573    /// up or went away) and every target is still where it was. Each question
574    /// takes tens of milliseconds; a full reading takes over a second. `None`
575    /// sends the caller to a full reading.
576    async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
577        let age = self.reading_age()?;
578        if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
579            return None;
580        }
581        let (_, dump, _) = self.ax.as_ref()?.latest()?;
582        let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
583        let ax = self.ax.as_mut()?;
584        for app in &dump.applications {
585            let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
586            let now: Vec<ObjKey> = ax
587                .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
588                .await?
589                .as_array()?
590                .iter()
591                .filter_map(ObjKey::from_ka)
592                .collect();
593            if now != was {
594                trace(|| "locate: the screen has changed since the last reading".into());
595                return None;
596            }
597        }
598        for (_, _, el) in &located.1 {
599            let el = el.as_ref()?;
600            let now = ax
601                .ask(el.key?, attrs::FRAME, ASK_WAIT)
602                .await?
603                .get("NS.rectval")
604                .and_then(|v| v.as_str())
605                .and_then(Rect::parse)?;
606            let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
607            if el.frame.as_ref().map(round) != Some(round(&now)) {
608                trace(|| "locate: a target has moved since the last reading".into());
609                return None;
610            }
611        }
612        trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
613        Some(located)
614    }
615
616    /// Locate the targets, make sure input can be sent, and locate them again
617    /// if getting ready took long enough for the screen to have moved on: a
618    /// cold input session takes seconds to open, and a tap must never go to
619    /// where something used to be.
620    async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
621        let located = match self.locate_confirmed(targets).await {
622            Some(l) => l,
623            None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
624        };
625        let started = std::time::Instant::now();
626        self.ensure_input().await?;
627        if started.elapsed() < Duration::from_millis(400) {
628            return Ok(located);
629        }
630        self.locate(targets, Fresh::Force).await
631    }
632
633    pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
634        if args.double && args.long_ms.is_some() {
635            return Err(Fail::invalid("--double and --long cannot be combined"));
636        }
637        if let Some(ms) = args.long_ms
638            && !(1..=60_000).contains(&ms)
639        {
640            return Err(Fail::invalid("--long must be 1..60000 ms"));
641        }
642        let target = parse_target(&args.target)?;
643        let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
644            Ok(located) => located,
645            Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
646        };
647        let (x, y, element) = points.remove(0);
648        let element_json = element.as_ref().map(brief);
649
650        let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
651        let before = screen.signature();
652        let input_began = Instant::now();
653        let gesture = if args.double {
654            Gesture::DoubleTap { x: nx, y: ny }
655        } else {
656            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
657        };
658        self.send_gesture(gesture).await?;
659        let after = if args.no_wait {
660            self.observe(input_began, AFTER_TAP).await
661        } else {
662            self.observe_settled(input_began, &before).await
663        };
664        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
665        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
666        if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
667            // Something of the system's is over the app now.
668            self.input_suspect = true;
669        }
670
671        let mut out = fields! {
672            "sent": true,
673            "verified": changed.is_some(),
674            "screen_changed": changed.unwrap_or(false),
675            "x": (x * 10.0).round() / 10.0,
676            "y": (y * 10.0).round() / 10.0,
677        };
678        if let Some(e) = element_json {
679            out.insert("element".into(), e);
680        }
681        if changed.is_none() {
682            out.insert(
683                "note".into(),
684                json!("the tap was sent but no reading came back to compare"),
685            );
686        }
687        if let Some(change) = app_change {
688            out.insert("app_changed".into(), json!(true));
689            out.insert("app".into(), change);
690        }
691        // The screen it waited for, so that no reading has to follow.
692        if !args.no_wait
693            && let Some((seq, screen)) = after {
694                out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
695                out.insert("apps".into(), json!(screen.apps));
696                out.insert("count".into(), json!(screen.elements.len()));
697                out.insert("seq".into(), json!(seq));
698                out.insert("elements".into(), json!(screen.elements));
699            }
700        Ok(out)
701    }
702
703    /// The last known screen size, when `read_failed` is the phone's
704    /// accessibility daemon giving no reading on an unlocked phone. Control
705    /// Center does that while a screen recording runs, until it is closed
706    /// (2026-11-12, over Wi-Fi and USB alike), and closing it takes a tap, a
707    /// swipe or a button: input that needs no element then goes out on this
708    /// size, unverified.
709    fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
710        if !daemon_silent(read_failed) {
711            return None;
712        }
713        self.screen_size
714    }
715
716    /// A tap on a point while the screen cannot be read (`unread_size`).
717    /// Nothing is read afterwards either: that would only wait out another
718    /// timeout.
719    async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
720        let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
721            return Err(read_failed);
722        };
723        let (x, y) = (*x, *y);
724        if x < 0.0 || y < 0.0 || x > w || y > h {
725            return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
726        }
727        let (nx, ny) = (normalize(x, w), normalize(y, h));
728        let gesture = if args.double {
729            Gesture::DoubleTap { x: nx, y: ny }
730        } else {
731            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
732        };
733        self.send_gesture(gesture).await?;
734        Ok(fields! {
735            "sent": true,
736            "verified": false,
737            "screen_changed": false,
738            "x": (x * 10.0).round() / 10.0,
739            "y": (y * 10.0).round() / 10.0,
740            "note": UNREAD_NOTE,
741        })
742    }
743
744    /// Tap several targets in a row on the screen as it is now: every target
745    /// is found before the first tap goes
746    /// out, nothing is re-read in between, and the screen is read once at the
747    /// end. For keypads and other screens that hold still while tapped.
748    pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
749        if targets.is_empty() {
750            return Err(Fail::invalid("the tap list is empty"));
751        }
752        let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
753        let (screen, points) = self.locate_ready(&parsed).await?;
754        let before = screen.signature();
755        let normalized: Vec<(u16, u16)> = points
756            .iter()
757            .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
758            .collect();
759        let tapped: Vec<Value> = points
760            .iter()
761            .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
762            .collect();
763        let input_began = Instant::now();
764        for (i, (nx, ny)) in normalized.iter().enumerate() {
765            let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
766            if let Err(mut fail) = self.send_gesture(tap).await {
767                if i > 0 {
768                    // The earlier taps of the list did go out.
769                    fail.outcome = crate::reply::Outcome::Unknown;
770                }
771                return Err(fail
772                    .hint(t(
773                        "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
774                        "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
775                    ))
776                    .with("taps_sent", i));
777            }
778        }
779        let after = self.observe_with(input_began, AFTER_TAP, true).await;
780        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
781        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
782        let mut out = fields! {
783            "sent": true,
784            "verified": changed.is_some(),
785            "screen_changed": changed.unwrap_or(false),
786            "taps": tapped.len(),
787            "elements": tapped,
788        };
789        if let Some(change) = app_change {
790            out.insert("app_changed".into(), json!(true));
791            out.insert("app".into(), change);
792        }
793        Ok(out)
794    }
795
796    /// Drag between two points given in logical points and report whether the
797    /// screen changed.
798    pub(crate) async fn drag_points(
799        &mut self,
800        from: (f64, f64),
801        to: (f64, f64),
802        duration: Duration,
803        motion: Motion,
804    ) -> Res<(bool, Option<Screen>)> {
805        let (_, screen) = self
806            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
807            .await?;
808        if screen.width <= 0.0 || screen.height <= 0.0 {
809            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
810        }
811        for (x, y) in [from, to] {
812            if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
813                return Err(Fail::invalid(format!(
814                    "point {x},{y} is outside the {}x{} screen",
815                    screen.width, screen.height
816                )));
817            }
818        }
819        let before = screen.signature();
820        let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
821        let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
822        let input_began = Instant::now();
823        self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
824        let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
825        if motion == Motion::Flick {
826            // The content coasts after a flick: read until two readings agree.
827            for _ in 0..3 {
828                let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
829                let still = match (&after, &next) {
830                    (Some(a), Some(b)) => a.signature() == b.signature(),
831                    _ => true,
832                };
833                after = next.or(after);
834                if still {
835                    break;
836                }
837            }
838        }
839        let changed = after.as_ref().is_some_and(|s| s.signature() != before);
840        Ok((changed, after))
841    }
842
843    async fn screen_size(&mut self) -> Res<(f64, f64)> {
844        let (_, screen) = self
845            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
846            .await?;
847        Ok((screen.width, screen.height))
848    }
849
850    pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
851        let read = self
852            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
853            .await;
854        let (w, h) = match &read {
855            Ok((_, screen)) => (screen.width, screen.height),
856            Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
857        };
858        let (from, to) = match (&args.direction, args.from, args.to) {
859            (Some(dir), None, None) => {
860                // The finger moves in the named direction across the middle.
861                let (cx, cy) = (w / 2.0, h / 2.0);
862                match dir.as_str() {
863                    "up" => ((cx, h * 0.70), (cx, h * 0.30)),
864                    "down" => ((cx, h * 0.30), (cx, h * 0.70)),
865                    "left" => ((w * 0.80, cy), (w * 0.20, cy)),
866                    "right" => ((w * 0.20, cy), (w * 0.80, cy)),
867                    other => {
868                        return Err(Fail::invalid(format!(
869                            "unknown direction `{other}` (up, down, left, right)"
870                        )));
871                    }
872                }
873            }
874            (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
875            _ => return Err(Fail::invalid("give a direction, or both from and to")),
876        };
877        let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
878        let motion = if args.drag { Motion::Place } else { Motion::Flick };
879        if read.is_err() {
880            // `unread_size`: drag on the last known size, read nothing after.
881            for (x, y) in [from, to] {
882                if x < 0.0 || y < 0.0 || x > w || y > h {
883                    return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
884                }
885            }
886            let a = (normalize(from.0, w), normalize(from.1, h));
887            let b = (normalize(to.0, w), normalize(to.1, h));
888            self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
889            return Ok(fields! {
890                "sent": true,
891                "verified": false,
892                "screen_changed": false,
893                "from": [from.0, from.1],
894                "to": [to.0, to.1],
895                "note": UNREAD_NOTE,
896            });
897        }
898        let (changed, after) = self.drag_points(from, to, duration, motion).await?;
899        Ok(fields! {
900            "sent": true,
901            "verified": after.is_some(),
902            "screen_changed": changed,
903            "from": [from.0, from.1],
904            "to": [to.0, to.1],
905        })
906    }
907
908    /// Move the content one step. "down" shows what is further down.
909    async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
910        let (w, h) = self.screen_size().await?;
911        let cx = w / 2.0;
912        let (from, to) = match direction {
913            "down" => ((cx, h * 0.68), (cx, h * 0.32)),
914            "up" => ((cx, h * 0.32), (cx, h * 0.68)),
915            other => {
916                return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
917            }
918        };
919        // A drag that slows to a stop moves the list by the finger's distance,
920        // without a fling that would overshoot.
921        self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
922            .await
923    }
924
925    pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
926        if args.direction != "up" && args.direction != "down" {
927            return Err(Fail::invalid("direction must be up or down"));
928        }
929        let Some(until) = &args.until else {
930            let (changed, after) = self.scroll_step(&args.direction).await?;
931            return Ok(fields! {
932                "sent": true,
933                "verified": after.is_some(),
934                "screen_changed": changed,
935                "scrolls": 1,
936            });
937        };
938        let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
939        let max = args.max.unwrap_or(10).clamp(1, 100);
940        let full = sel.needs_identifier();
941        let mut scrolls = 0u32;
942        let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
943        loop {
944            let matches = dedupe_targets(sel.find(&screen));
945            // Found means: on screen and tappable (not under a toolbar).
946            if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
947                if matches.len() > 1 {
948                    return Err(Fail::not_sent(
949                        "ambiguous_selector",
950                        format!("{} elements match {}", matches.len(), sel.describe()),
951                    )
952                    .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
953                    .with("scrolls", scrolls));
954                }
955                return Ok(fields! {
956                    "found": true,
957                    "scrolls": scrolls,
958                    "screen_changed": scrolls > 0,
959                    "element": brief(el),
960                });
961            }
962            if scrolls >= max {
963                return Err(Fail::new(
964                    crate::reply::Outcome::Done,
965                    "not_found",
966                    format!("{} not found after {scrolls} scrolls", sel.describe()),
967                )
968                .with("found", false)
969                .with("scrolls", scrolls)
970                .with("screen_summary", screen.summary(12)));
971            }
972            let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
973            scrolls += 1;
974            if !changed {
975                // One still screen can be a dropped drag; two in a row is the
976                // end of the list.
977                (changed, after) = self.scroll_step(&args.direction).await?;
978                scrolls += 1;
979            }
980            if !changed {
981                return Err(Fail::new(
982                    crate::reply::Outcome::Done,
983                    "not_found",
984                    format!(
985                        "{} not found; the list stopped moving after {scrolls} scrolls",
986                        sel.describe()
987                    ),
988                )
989                .with("found", false)
990                .with("scrolls", scrolls)
991                .with("screen_summary", screen.summary(12)));
992            }
993            screen = match (after, full) {
994                (Some(s), false) => s,
995                _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
996            };
997        }
998    }
999
1000    pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
1001        let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
1002        let (_, screen) = self
1003            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1004            .await?;
1005        let before = Some(screen.signature());
1006        let input_began = Instant::now();
1007        let input = self.ensure_input().await?;
1008        if let Err(e) = input.key(usage, &modifiers).await {
1009            self.drop_input();
1010            return Err(Fail::unknown("input_failed", format!("{e:#}")));
1011        }
1012        let changed = match before {
1013            Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
1014            None => None,
1015        };
1016        Ok(fields! {
1017            "sent": true,
1018            "verified": changed.is_some(),
1019            "screen_changed": changed.unwrap_or(false),
1020            "key": args.key,
1021        })
1022    }
1023
1024    pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
1025        let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
1026        // A button needs no screen size; the reading is only to compare with.
1027        let before = match self
1028            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1029            .await
1030        {
1031            Ok((_, screen)) => Some(screen.signature()),
1032            Err(read_failed) if daemon_silent(&read_failed) => None,
1033            Err(read_failed) => return Err(read_failed),
1034        };
1035        let input_began = Instant::now();
1036        self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
1037        let changed = match &before {
1038            Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
1039            None => None,
1040        };
1041        let mut out = fields! {
1042            "sent": true,
1043            "verified": changed.is_some(),
1044            "screen_changed": changed.unwrap_or(false),
1045            "button": args.button,
1046        };
1047        if before.is_none() {
1048            out.insert("note".into(), json!(UNREAD_NOTE));
1049        }
1050        Ok(out)
1051    }
1052
1053    pub async fn cmd_apps(&mut self) -> Res {
1054        let apps = self.list_apps(true).await?;
1055        let list: Vec<Value> = apps
1056            .iter()
1057            .map(|a| {
1058                json!({
1059                    "bundle_id": a.bundle_identifier,
1060                    "name": a.name,
1061                    "version": a.version,
1062                    "first_party": a.is_first_party,
1063                    "removable": a.is_removable,
1064                })
1065            })
1066            .collect();
1067        Ok(fields! { "count": list.len(), "apps": list })
1068    }
1069
1070    async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
1071        // Something with a dot and no spaces is a bundle id; anything else is
1072        // looked up by name.
1073        if app.contains('.') && !app.contains(' ') {
1074            return Ok(app.to_string());
1075        }
1076        let want = nfc(app).to_lowercase();
1077        let apps = self.list_apps(false).await?;
1078        let hits: Vec<&idevice::core_device::AppListEntry> = apps
1079            .iter()
1080            .filter(|a| nfc(&a.name).to_lowercase() == want)
1081            .collect();
1082        match hits.len() {
1083            1 => Ok(hits[0].bundle_identifier.clone()),
1084            0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
1085                .hint(t(
1086                    "`iph apps`로 이름과 bundle id를 확인하세요.",
1087                    "See names and bundle ids with `iph apps`.",
1088                ))),
1089            _ => Err(Fail::not_sent(
1090                "ambiguous_app",
1091                format!("{} apps are named \"{app}\"", hits.len()),
1092            )
1093            .with(
1094                "candidates",
1095                Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
1096            )),
1097        }
1098    }
1099
1100    pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
1101        let bundle = self.resolve_bundle(&args.app).await?;
1102        let mut svc = self.app_service().await?;
1103        let launch_began = Instant::now();
1104        // From here the launch request may have reached the phone.
1105        let launched = tokio::time::timeout(
1106            Duration::from_secs(30),
1107            svc.launch_application(bundle.as_str(), &[], args.restart, false, None, None, None),
1108        )
1109        .await
1110        .map_err(|_| "no answer within 30 s".to_string())
1111        .and_then(|r| r.map_err(|e| format!("{e:?}")));
1112        let resp = match launched {
1113            Ok(r) => r,
1114            Err(msg) => {
1115                let locked = msg.to_lowercase().contains("lock");
1116                return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1117                    t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1118                } else {
1119                    t(
1120                        "bundle id가 맞는지 `iph apps`로 확인하세요.",
1121                        "Check the bundle id with `iph apps`.",
1122                    )
1123                }));
1124            }
1125        };
1126        let pid = resp.pid as i64;
1127        // The app is in front when a reading lists it as an application.
1128        let mut foreground = false;
1129        let mut delay = AFTER_LAUNCH;
1130        for _ in 0..4 {
1131            if let Some((_, screen)) = self.observe(launch_began, delay).await
1132                && screen.apps.iter().any(|a| a.pid == pid) {
1133                    foreground = true;
1134                    break;
1135                }
1136            delay = Duration::from_millis(300);
1137        }
1138        Ok(fields! {
1139            "sent": true,
1140            "verified": foreground,
1141            "foreground": foreground,
1142            "bundle_id": bundle,
1143            "pid": pid,
1144        })
1145    }
1146
1147    pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1148        let png = self.screenshot_png().await?;
1149        let path = std::path::PathBuf::from(&args.path);
1150        if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1151            std::fs::create_dir_all(dir)
1152                .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1153        }
1154        std::fs::write(&path, &png)
1155            .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1156        Ok(fields! { "path": args.path, "bytes": png.len() })
1157    }
1158
1159    /// The text element that has the keyboard, if exactly one does.
1160    fn focused_field(screen: &Screen) -> Option<&Element> {
1161        let mut focused = screen.elements.iter().filter(|e| e.focused);
1162        let first = focused.next()?;
1163        focused.next().is_none().then_some(first)
1164    }
1165
1166    /// What a field shows after a paste, asked of the phone directly: again
1167    /// every 0.1 s until it differs from before or
1168    /// `PASTE_SHOWS` has passed. `None` when the phone does not answer, and
1169    /// the caller reads the whole screen instead.
1170    async fn field_value(&mut self, field: &Element) -> Option<String> {
1171        let key = field.key?;
1172        let before = field.value.as_deref();
1173        let began = Instant::now();
1174        loop {
1175            let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1176            let now = crate::ax::tree::value_text(Some(&value))?;
1177            if Some(now.as_str()) != before || began.elapsed() >= PASTE_SHOWS {
1178                return Some(now);
1179            }
1180            tokio::time::sleep(Duration::from_millis(100)).await;
1181        }
1182    }
1183
1184    pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1185        if args.text.is_empty() && !args.clear {
1186            return Err(Fail::invalid("nothing to type"));
1187        }
1188        if args.text.len() > 1024 * 1024 {
1189            return Err(Fail::invalid("text is larger than 1 MiB"));
1190        }
1191        let (_, screen) = self
1192            .screen(Fresh::Force, false, BuildOptions::default())
1193            .await?;
1194        let Some(field) = Self::focused_field(&screen).cloned() else {
1195            return Err(Fail::not_sent(
1196                "no_focused_field",
1197                "no text field has the keyboard right now",
1198            )
1199            .hint(t(
1200                "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1201                "Tap the field first with `iph tap` so the keyboard comes up.",
1202            ))
1203            .with("screen_summary", screen.summary(12)));
1204        };
1205        trace(|| "type: focused field found".into());
1206
1207        // The input session first: if it cannot be had, nothing has been
1208        // touched yet, the clipboard included.
1209        self.ensure_input().await?;
1210        let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1211
1212        // Remember what the user had on the clipboard and, meanwhile, register
1213        // the temporary keyboard (no key goes out yet).
1214        let input = self.ensure_input().await?;
1215        let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1216        trace(|| "type: clipboard saved, keyboard attached".into());
1217        if let Err(e) = attached {
1218            // Nothing has been touched: the clipboard still holds the user's content.
1219            return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1220        }
1221        if !args.text.is_empty()
1222            && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1223                let _ = input.keyboard_detach().await;
1224                self.invalidate_readings();
1225                return Err(Fail::not_sent("clipboard_failed", e));
1226            }
1227        trace(|| "type: clipboard set".into());
1228
1229        let cmd = vec![crate::keys::MOD_COMMAND];
1230        let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1231        if args.clear {
1232            combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1233            combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1234        }
1235        if !args.text.is_empty() {
1236            combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1237        }
1238        let keys_began = Instant::now();
1239        let pressed = input.keyboard_press(&combos).await;
1240        trace(|| "type: keys sent".into());
1241        if let Err(e) = pressed {
1242            let _ = input.keyboard_detach().await;
1243            self.drop_input();
1244            self.invalidate_readings();
1245            let restored = saved.restore(&adapter, port, &args.text).await;
1246            return Err(Fail::unknown("input_failed", format!("{e:#}"))
1247                .hint(t(
1248                    "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1249                    "The input may have gone out. Do not resend; check the screen with `iph read`.",
1250                ))
1251                .with("clipboard_restored", restored));
1252        }
1253
1254        // The paste keys have gone out, so the clipboard can go back while
1255        // the field is read. The temporary keyboard stays attached meanwhile:
1256        // the screen holds still and Return can follow at once. What the field
1257        // shows is reported, not judged: apps format, mask, correct and clear
1258        // what was pasted (₩ 30,000 for 30000), so only the caller can tell
1259        // whether it is right, as with Playwright's fill().
1260        let restoring = saved.restore(&adapter, port, &args.text);
1261        let reading = async {
1262            tokio::time::sleep(AFTER_PASTE).await;
1263            if let Some(value) = self.field_value(&field).await {
1264                return Some(value);
1265            }
1266            let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1267            // Focus may have moved to a permission dialog or another field.
1268            // Only the original field can tell us what this paste changed.
1269            screen.by_id(&field.id).and_then(|e| e.value.clone())
1270        };
1271        let (restored, value) = tokio::join!(restoring, reading);
1272        trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1273        let observation = TypeObservation::new(&args, field.value.clone(), value);
1274
1275        let mut submitted = Some(false);
1276        let mut submit_error = None;
1277        if args.submit && observation.can_submit() {
1278            if let Some(input) = self.input.as_mut() {
1279                match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1280                    Ok(()) => submitted = Some(true),
1281                    Err(e) => {
1282                        submitted = None;
1283                        submit_error = Some(format!("{e:#}"));
1284                    }
1285                }
1286            } else {
1287                submit_error = Some("the input session closed before Return could be sent".into());
1288            }
1289        }
1290        if let Some(input) = self.input.as_mut()
1291            && input.keyboard_detach().await.is_err() {
1292                self.drop_input();
1293            }
1294        // The on-screen keyboard comes back now; readings from before show
1295        // the screen without it.
1296        self.invalidate_readings();
1297
1298        if submit_error.is_some() {
1299            self.drop_input();
1300        }
1301        // Compare complete readings after the temporary keyboard is gone.
1302        // A field read alone cannot tell whether the rest of the screen moved.
1303        let screen_changed = self
1304            .observe(keys_began, AFTER_PASTE)
1305            .await
1306            .map(|(_, after)| after.signature() != screen.signature());
1307        let mut result = observation.result(
1308            &args,
1309            &field,
1310            restored,
1311            submitted,
1312            screen_changed,
1313            submit_error,
1314        );
1315        if !restored && saved.unreadable() {
1316            let out = match &mut result {
1317                Ok(out) => out,
1318                Err(fail) => &mut fail.extra,
1319            };
1320            out.insert(
1321                "clipboard_note".into(),
1322                json!(t(
1323                    "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1324                    "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1325                )),
1326            );
1327        }
1328        result
1329    }
1330}
1331
1332const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1333
1334/// Kinds of data the phone's clipboard service holds itself: what `iph type`
1335/// puts there, and plain text or a picture. An item that carries any other kind
1336/// (rich text, an app's own format) was copied in an app, and its bytes are
1337/// only promised: the service asks that app for them when someone reads them.
1338/// Read from here, that request never came back. Text copied in Notes stalled
1339/// past 8 s for every kind on offer (rtf and plain text alike), and the
1340/// service then answered nothing at all, writes included, for about a minute
1341/// (2026-11-11: `iph type` right after ⌘C in Notes failed twice with "set
1342/// clipboard: no answer" and lost the clipboard the third time). Such content
1343/// is not read.
1344const SERVICE_KINDS: [&str; 6] = [
1345    "public.utf8-plain-text",
1346    "public.plain-text",
1347    "public.text",
1348    "public.png",
1349    "public.jpeg",
1350    "public.tiff",
1351];
1352/// What the service's own items took to read: 75 ~ 124 ms.
1353const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1354
1355/// Whether an item with these kinds was copied in an app, so reading its
1356/// bytes would wait on that app.
1357fn app_provided(types: &[String]) -> bool {
1358    types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1359}
1360
1361/// What the clipboard holds, as far as it can be read without waiting on an app.
1362#[derive(Debug, PartialEq)]
1363enum Peek {
1364    Empty,
1365    Text(String),
1366    Image(String, Vec<u8>),
1367    /// Copied in an app; left unread (see `SERVICE_KINDS`).
1368    AppContent,
1369    /// No answer, or something not understood.
1370    Unknown,
1371}
1372
1373/// What the user had on the phone's clipboard before `iph type` borrowed it.
1374struct Clipboard {
1375    before: Peek,
1376}
1377
1378impl Clipboard {
1379    /// Every call uses its own connection, so a late answer to one request
1380    /// cannot be taken for the answer to the next.
1381    async fn connect(
1382        adapter: &crate::tcp::handle::AdapterHandle,
1383        port: u16,
1384    ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1385        use idevice::RsdService;
1386        let mut adapter = adapter.clone();
1387        let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1388            .await
1389            .map_err(|_| "clipboard service: no answer".to_string())?
1390            .map_err(|e| format!("clipboard service: {e}"))?;
1391        idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1392            .await
1393            .map_err(|e| format!("clipboard service: {e:?}"))
1394    }
1395
1396    /// Look at the clipboard. Bytes come along only as promises, and only
1397    /// those the service holds itself are fetched.
1398    async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1399        use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1400        let general = idevice::core_device::GENERAL_PASTEBOARD;
1401        let Ok(mut clip) = Self::connect(adapter, port).await else {
1402            return Peek::Unknown;
1403        };
1404        let pulled = tokio::time::timeout(
1405            Duration::from_secs(8),
1406            clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1407        )
1408        .await;
1409        let Ok(Ok(snap)) = pulled else {
1410            return Peek::Unknown;
1411        };
1412        trace(|| {
1413            let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1414            format!("clipboard: kinds [{}]", kinds.join(" | "))
1415        });
1416        let Some(item) = snap.items.first() else {
1417            return Peek::Empty;
1418        };
1419        if snap.items.iter().any(|i| app_provided(&i.types)) {
1420            return Peek::AppContent;
1421        }
1422        // Text before a picture, in the order of `SERVICE_KINDS`.
1423        for kind in SERVICE_KINDS {
1424            let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1425                continue;
1426            };
1427            let bytes = match &entry.payload {
1428                PasteboardPayload::Inline(b) => Some(b.clone()),
1429                PasteboardPayload::Promised { .. } => {
1430                    match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1431                        Ok(Ok(b)) => b,
1432                        Ok(Err(_)) => None,
1433                        // Whatever holds it up may hold up the service too.
1434                        Err(_) => return Peek::Unknown,
1435                    }
1436                }
1437                PasteboardPayload::Error(_) => None,
1438            };
1439            let Some(bytes) = bytes else { continue };
1440            return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1441                Peek::Image(kind.to_string(), bytes)
1442            } else {
1443                match String::from_utf8(bytes) {
1444                    Ok(text) => Peek::Text(text),
1445                    Err(_) => continue,
1446                }
1447            };
1448        }
1449        Peek::Unknown
1450    }
1451
1452    async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1453        let mut clip = Self::connect(adapter, port).await?;
1454        let general = idevice::core_device::GENERAL_PASTEBOARD;
1455        match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1456            Ok(Ok(())) => Ok(()),
1457            Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1458            Err(_) => Err("set clipboard: no answer".into()),
1459        }
1460    }
1461
1462    async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1463        Clipboard { before: Self::peek(adapter, port).await }
1464    }
1465
1466    /// True when what was on the clipboard could not be read and is lost
1467    /// once `iph type` puts its text there.
1468    fn unreadable(&self) -> bool {
1469        matches!(self.before, Peek::AppContent | Peek::Unknown)
1470    }
1471
1472    /// Put the old content back. True when the clipboard is known to hold
1473    /// what it held before.
1474    async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1475        if typed.is_empty() {
1476            return true;
1477        }
1478        // Give the paste a moment to read the clipboard.
1479        tokio::time::sleep(Duration::from_millis(150)).await;
1480        // Only touch the clipboard if it still holds what we put there; the
1481        // user may have copied something else meanwhile.
1482        if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1483            return false;
1484        }
1485        let general = idevice::core_device::GENERAL_PASTEBOARD;
1486        let wrote = match &self.before {
1487            Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1488            Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1489                Ok(mut clip) => {
1490                    matches!(
1491                        tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1492                        Ok(Ok(()))
1493                    )
1494                }
1495                Err(_) => false,
1496            },
1497            // The clipboard was empty, or what was there is gone: either way
1498            // the typed text does not stay behind.
1499            Peek::Empty | Peek::AppContent | Peek::Unknown => {
1500                let _ = Self::set_text(adapter, port, "").await;
1501                return matches!(self.before, Peek::Empty);
1502            }
1503        };
1504        if !wrote {
1505            return false;
1506        }
1507        match (Self::peek(adapter, port).await, &self.before) {
1508            (Peek::Text(now), Peek::Text(old)) => &now == old,
1509            (Peek::Image(..), Peek::Image(..)) => true,
1510            _ => false,
1511        }
1512    }
1513}
1514
1515#[cfg(test)]
1516mod type_tests {
1517    use super::*;
1518    use crate::reply::envelope;
1519
1520    fn field(before: Option<&str>) -> Element {
1521        serde_json::from_value(json!({
1522            "id": "original", "role": "text_field", "value": before, "focused": true,
1523        }))
1524        .unwrap()
1525    }
1526
1527    fn result(
1528        args: &TypeArgs,
1529        before: Option<&str>,
1530        after: Option<&str>,
1531        screen: Option<bool>,
1532    ) -> Value {
1533        let field = field(before);
1534        let observed = TypeObservation::new(args, field.value.clone(), after.map(String::from));
1535        let submit = args.submit && observed.can_submit();
1536        envelope(
1537            observed.result(args, &field, true, Some(submit), screen, None),
1538            Instant::now(),
1539        )
1540    }
1541
1542    #[test]
1543    fn changed_values_are_observations_not_exact_text_matches() {
1544        for value in ["30000", "₩ 30,000", "•••••", "corrected text", " \n "] {
1545            let args = TypeArgs {
1546                text: "30000".into(),
1547                submit: true,
1548                ..Default::default()
1549            };
1550            let out = result(&args, Some(""), Some(value), Some(true));
1551            assert_eq!(out["ok"], true, "{out}");
1552            assert_eq!(out["outcome"], "done");
1553            assert_eq!(out["verified"], true);
1554            assert_eq!(out["value_before"], "");
1555            assert_eq!(out["value"], value);
1556            assert_eq!(out["value_changed"], true);
1557            assert_eq!(out["screen_changed"], true);
1558            assert_eq!(out["submit_sent"], true);
1559            assert!(out.get("warning").is_none());
1560        }
1561    }
1562
1563    #[test]
1564    fn unchanged_paste_warns_without_claiming_permission_was_denied() {
1565        let args = TypeArgs {
1566            text: "테스트 문장".into(),
1567            ..Default::default()
1568        };
1569        for value in ["", "old text", "•••••"] {
1570            let out = result(&args, Some(value), Some(value), Some(false));
1571            assert_eq!(out["ok"], true);
1572            assert_eq!(out["outcome"], "done");
1573            assert_eq!(out["sent"], true);
1574            assert_eq!(out["verified"], true, "verified still means read back");
1575            assert_eq!(out["value_changed"], false);
1576            assert_eq!(out["screen_changed"], false);
1577            assert_eq!(out["warning"]["code"], "value_unchanged");
1578            assert!(
1579                out["warning"]["message"]
1580                    .as_str()
1581                    .unwrap()
1582                    .contains("may not")
1583            );
1584            assert!(out["warning"]["hint"].as_str().is_some());
1585        }
1586        // A dialog or another part of the screen may change while the field does not.
1587        let out = result(&args, Some(""), Some(""), Some(true));
1588        assert_eq!(out["value_changed"], false);
1589        assert_eq!(out["screen_changed"], true);
1590        assert!(out.get("warning").is_some());
1591    }
1592
1593    #[test]
1594    fn unreadable_and_uncomparable_values_do_not_become_empty_strings() {
1595        let args = TypeArgs {
1596            text: "hello".into(),
1597            ..Default::default()
1598        };
1599        let out = result(&args, Some("old"), None, None);
1600        assert_eq!(out["ok"], true);
1601        assert_eq!(out["verified"], false);
1602        assert!(out["value"].is_null());
1603        assert!(out["value_changed"].is_null());
1604        assert!(out["screen_changed"].is_null());
1605        assert_eq!(out["warning"]["code"], "value_unreadable");
1606        let out = result(&args, None, Some("hello"), Some(true));
1607        assert_eq!(out["verified"], true);
1608        assert!(out["value_before"].is_null());
1609        assert!(out["value_changed"].is_null());
1610        assert_eq!(out["warning"]["code"], "value_change_unknown");
1611        // A readable field does not imply a complete screen comparison succeeded.
1612        let out = result(&args, Some(""), Some("hello"), None);
1613        assert_eq!(out["verified"], true);
1614        assert!(out["screen_changed"].is_null());
1615    }
1616
1617    #[test]
1618    fn submit_is_withheld_when_the_input_effect_needs_review() {
1619        let args = TypeArgs {
1620            text: "hello".into(),
1621            submit: true,
1622            ..Default::default()
1623        };
1624        for (before, after, code) in [
1625            (Some(""), Some(""), "value_unchanged"),
1626            (Some("old"), Some("old"), "value_unchanged"),
1627            (Some("old"), None, "value_unreadable"),
1628            (None, Some("hello"), "value_change_unknown"),
1629            (Some("old"), Some(""), "value_empty"),
1630        ] {
1631            let out = result(&args, before, after, Some(false));
1632            assert_eq!(out["ok"], false, "{out}");
1633            assert_eq!(out["outcome"], "unknown");
1634            assert_eq!(out["sent"], true);
1635            assert_eq!(out["submit_sent"], false);
1636            assert_eq!(out["error"]["code"], "type_not_verified");
1637            assert_eq!(out["warning"]["code"], code);
1638            assert_eq!(out["value"], json!(after));
1639            assert_eq!(out["clipboard_restored"], true);
1640        }
1641    }
1642
1643    #[test]
1644    fn clear_and_identical_replacement_are_reported_separately() {
1645        let mut args = TypeArgs {
1646            clear: true,
1647            submit: true,
1648            ..Default::default()
1649        };
1650        for before in [Some(""), Some("old"), None] {
1651            let out = result(&args, before, Some(""), Some(false));
1652            assert_eq!(out["ok"], true);
1653            assert_eq!(out["submit_sent"], true);
1654            assert!(out.get("warning").is_none());
1655        }
1656        let out = result(&args, Some("old"), Some("old"), Some(false));
1657        assert_eq!(out["warning"]["code"], "clear_not_observed");
1658        assert_eq!(out["submit_sent"], false);
1659        // Replacing with identical text can be intentional, but it does not
1660        // establish whether paste worked. The caller must inspect before Return.
1661        args.text = "same".into();
1662        let out = result(&args, Some("same"), Some("same"), Some(false));
1663        assert_eq!(out["outcome"], "unknown");
1664        assert_eq!(out["warning"]["code"], "value_unchanged");
1665    }
1666
1667    #[test]
1668    fn failed_return_keeps_observations_and_does_not_claim_it_was_unsent() {
1669        let args = TypeArgs {
1670            text: "hello".into(),
1671            submit: true,
1672            ..Default::default()
1673        };
1674        let field = field(Some(""));
1675        let observed = TypeObservation::new(&args, field.value.clone(), Some("hello".into()));
1676        let out = envelope(
1677            observed.result(
1678                &args,
1679                &field,
1680                false,
1681                None,
1682                None,
1683                Some("write failed".into()),
1684            ),
1685            Instant::now(),
1686        );
1687        assert_eq!(out["outcome"], "unknown");
1688        assert_eq!(out["error"]["code"], "input_failed");
1689        assert!(out["submit_sent"].is_null());
1690        assert_eq!(out["value_changed"], true);
1691        assert_eq!(out["clipboard_restored"], false);
1692    }
1693}
1694
1695#[cfg(test)]
1696mod clipboard_tests {
1697    use super::app_provided;
1698
1699    fn kinds(k: &[&str]) -> Vec<String> {
1700        k.iter().map(|s| s.to_string()).collect()
1701    }
1702
1703    #[test]
1704    fn notes_copy_is_app_content() {
1705        // What ⌘C in Notes put on the clipboard (2026-11-11).
1706        assert!(app_provided(&kinds(&[
1707            "public.utf8-plain-text",
1708            "com.apple.notes.richtext",
1709            "com.apple.webarchive",
1710            "com.apple.flat-rtfd",
1711            "iOS rich content paste pasteboard type",
1712            "public.html",
1713            "net.daringfireball.markdown",
1714            "public.rtf",
1715        ])));
1716    }
1717
1718    #[test]
1719    fn what_iph_puts_there_is_not() {
1720        // `set_text` writes these three kinds.
1721        assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1722        assert!(!app_provided(&kinds(&["public.png"])));
1723    }
1724}