Skip to main content

iphone_cli/
actions.rs

1//! The commands themselves, shared by the CLI, the MCP server, `batch` and
2//! scripts. Each takes JSON arguments and returns the fields of its result.
3
4use std::time::{Duration, Instant};
5
6use serde::Deserialize;
7use serde_json::{Value, json};
8use unicode_normalization::UnicodeNormalization;
9
10use crate::{
11    ax::{
12        attrs,
13        dump::ObjKey,
14        tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
15    },
16    device::{
17        ax::trace,
18        input::{Motion, normalize},
19        session::{Fresh, Gesture, Session, reuse},
20    },
21    fields,
22    i18n::t,
23    reply::{Fail, Res},
24    selector::{Selector, dedupe_targets},
25};
26
27/// How long after an input to ask for the new screen. Measured on iOS 27: a
28/// request made while a navigation animation still runs comes back about
29/// 0.4 s later than one made just after it (2026-11-10, 14 taps).
30pub const AFTER_TAP: Duration = Duration::from_millis(480);
31/// A drag that ends with the finger at rest only leaves a short bounce.
32pub const AFTER_DRAG: Duration = Duration::from_millis(300);
33pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
34pub const AFTER_PASTE: Duration = Duration::from_millis(200);
35/// How long a tap waits for the screen to start changing. A picker that
36/// another part of the app puts up came about 1.5 s after the tap
37/// (2026-11-12, 당근 '이미지 추가').
38pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
39/// How long after the tap a changed screen may take to hold still.
40pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
41/// A reading up to this old is used for a tap once the phone confirms, by a
42/// few quick questions, that the screen is still as read. Older ones are read
43/// again in full.
44const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
45/// How long a pasted text may take to show in its field.
46const PASTE_SHOWS: Duration = Duration::from_millis(1500);
47/// How long to wait for the answer to one quick question.
48const ASK_WAIT: Duration = Duration::from_millis(1000);
49/// An unasked dump this recently before an input means the app is one that
50/// triggers them; its readings are then double-checked.
51const UNASKED_MEMORY: Duration = Duration::from_secs(5);
52/// The result's note for an input sent while the screen could not be read.
53const UNREAD_NOTE: &str =
54    "the screen could not be read; sent on the last known screen size, not verified";
55
56#[derive(Debug, Default, Deserialize)]
57#[serde(default, deny_unknown_fields)]
58pub struct ReadArgs {
59    /// Only the foreground application's elements.
60    pub app: bool,
61    /// Include elements outside the screen.
62    pub all: bool,
63    /// Wait for the second pass so identifiers are present.
64    pub full: bool,
65    /// Attach every numbered attribute.
66    pub raw: bool,
67    /// Accept a cached reading up to this old instead of asking the phone.
68    pub max_age_ms: Option<u64>,
69}
70
71#[derive(Debug, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct FindArgs {
74    pub selector: Value,
75    #[serde(default)]
76    pub all: bool,
77}
78
79#[derive(Debug, Default, Deserialize)]
80#[serde(default, deny_unknown_fields)]
81pub struct TapArgs {
82    /// A selector (string or object), `"x,y"`, or `{x, y}`.
83    pub target: Value,
84    pub long_ms: Option<u64>,
85    pub double: bool,
86    /// Read the screen once right after the tap instead of waiting for it to
87    /// change and hold still.
88    pub no_wait: bool,
89}
90
91#[derive(Debug, Default, Deserialize)]
92#[serde(default, deny_unknown_fields)]
93pub struct SwipeArgs {
94    pub direction: Option<String>,
95    pub from: Option<[f64; 2]>,
96    pub to: Option<[f64; 2]>,
97    pub duration_ms: Option<u64>,
98    /// Slow to a stop before lifting (a drag) instead of flicking.
99    pub drag: bool,
100}
101
102#[derive(Debug, Default, Deserialize)]
103#[serde(default, deny_unknown_fields)]
104pub struct ScrollArgs {
105    pub direction: String,
106    pub until: Option<Value>,
107    pub max: Option<u32>,
108}
109
110#[derive(Debug, Default, Deserialize)]
111#[serde(default, deny_unknown_fields)]
112pub struct TypeArgs {
113    pub text: String,
114    pub clear: bool,
115    pub submit: bool,
116}
117
118#[derive(Debug, Deserialize)]
119#[serde(deny_unknown_fields)]
120pub struct KeyArgs {
121    pub key: String,
122}
123
124#[derive(Debug, Deserialize)]
125#[serde(deny_unknown_fields)]
126pub struct ButtonArgs {
127    pub button: String,
128}
129
130#[derive(Debug, Default, Deserialize)]
131#[serde(deny_unknown_fields)]
132pub struct LaunchArgs {
133    pub app: String,
134    /// Quit the app first, so it starts from its first screen.
135    #[serde(default)]
136    pub restart: bool,
137}
138
139#[derive(Debug, Deserialize)]
140#[serde(deny_unknown_fields)]
141pub struct ScreenshotArgs {
142    pub path: String,
143}
144
145pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
146    let v = if args.is_null() { json!({}) } else { args.clone() };
147    serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
148}
149
150pub fn brief(el: &Element) -> Value {
151    let mut v = json!({ "id": el.id, "role": el.role });
152    if let Some(l) = &el.label {
153        v["label"] = json!(l);
154    }
155    if let Some(val) = &el.value {
156        v["value"] = json!(val);
157    }
158    if let Some(i) = &el.identifier {
159        v["identifier"] = json!(i);
160    }
161    if let Some(f) = &el.frame {
162        v["frame"] = json!(f);
163    }
164    v
165}
166
167pub enum Target {
168    Selector(Selector),
169    Point(f64, f64),
170}
171
172/// The reading failed because the phone's accessibility daemon gave none on a
173/// phone not known to be locked (`Session::unread_size`).
174fn daemon_silent(read_failed: &Fail) -> bool {
175    let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
176    read_failed.code == "ax_timeout" && !locked
177}
178
179pub fn parse_target(v: &Value) -> Res<Target> {
180    if let Some(obj) = v.as_object()
181        && let (Some(x), Some(y)) = (
182            obj.get("x").and_then(|n| n.as_f64()),
183            obj.get("y").and_then(|n| n.as_f64()),
184        )
185            && obj.len() == 2 {
186                return Ok(Target::Point(x, y));
187            }
188    if let Some(s) = v.as_str() {
189        let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
190        if parts.len() == 2
191            && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
192                return Ok(Target::Point(x, y));
193            }
194    }
195    Selector::from_value(v)
196        .map(Target::Selector)
197        .map_err(|e| Fail::invalid(e.to_string()))
198}
199
200/// The single element a selector names, or an error that lists what matched.
201pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
202    let matches = dedupe_targets(sel.find(screen));
203    match matches.len() {
204        1 => Ok(matches[0]),
205        0 => Err(Fail::not_sent(
206            "not_found",
207            format!("no element matches {}", sel.describe()),
208        )
209        .hint(t(
210            "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
211            "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
212        ))
213        .with("selector", sel.describe())
214        .with("screen_summary", screen.summary(12))),
215        n => Err(Fail::not_sent(
216            "ambiguous_selector",
217            format!("{n} elements match {}", sel.describe()),
218        )
219        .hint(t(
220            "role=, within=, index= 으로 하나만 남게 좁히세요.",
221            "Narrow it to one with role=, within= or index=.",
222        ))
223        .with("selector", sel.describe())
224        .with(
225            "candidates",
226            Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
227        )),
228    }
229}
230
231/// A point inside the visible part of `el` that no other element covers.
232fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
233    let frame = el.frame.ok_or_else(|| {
234        Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
235    })?;
236    let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
237    let visible = frame.intersection(&bounds).ok_or_else(|| {
238        Fail::not_sent("off_screen", "the element is outside the screen")
239            .hint(t(
240                "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
241                "Bring it on screen with `iph scroll down --until …`.",
242            ))
243            .with("element", brief(el))
244    })?;
245    let covered = |x: f64, y: f64| -> Option<&Element> {
246        let hit = screen.element_at(x, y)?;
247        if hit.id == el.id {
248            return None;
249        }
250        // A child of the target (a label inside a row) is fine; an element
251        // that merely overlaps it (a toolbar over a list) is not.
252        let inside = hit.frame.is_some_and(|h| {
253            h.x >= frame.x - 1.0
254                && h.y >= frame.y - 1.0
255                && h.x + h.w <= frame.x + frame.w + 1.0
256                && h.y + h.h <= frame.y + frame.h + 1.0
257        });
258        let encloses = hit.frame.is_some_and(|h| {
259            frame.x >= h.x - 1.0
260                && frame.y >= h.y - 1.0
261                && frame.x + frame.w <= h.x + h.w + 1.0
262                && frame.y + frame.h <= h.y + h.h + 1.0
263        });
264        (!inside && !encloses).then_some(hit)
265    };
266    let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
267    let mut blocker = None;
268    for (fx, fy) in candidates {
269        let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
270        match covered(x, y) {
271            None => return Ok((x, y)),
272            Some(b) => blocker = blocker.or(Some(b)),
273        }
274    }
275    Err(Fail::not_sent("covered", "another element covers the target")
276        .hint(t(
277            "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
278            "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
279        ))
280        .with("element", brief(el))
281        .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
282}
283
284fn nfc(s: &str) -> String {
285    s.nfc().collect()
286}
287
288/// The app now in front, when it is not the one that was in front before.
289fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
290    let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
291    let now = after.apps.first()?;
292    (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
293}
294
295impl Session {
296    fn build_opts(all: bool, raw: bool) -> BuildOptions {
297        BuildOptions { include_offscreen: all, raw }
298    }
299
300    pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
301        let fresh = match args.max_age_ms {
302            Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
303            None => Fresh::Force,
304        };
305        let (seq, mut screen) = self
306            .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
307            .await?;
308        if args.app
309            && let Some(front) = screen.apps.first().map(|a| a.pid) {
310                screen.elements.retain(|e| e.pid == front);
311                screen.apps.retain(|a| a.pid == front);
312            }
313        let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
314        Ok(fields! {
315            "screen": {"width": screen.width, "height": screen.height},
316            "apps": screen.apps,
317            "count": screen.elements.len(),
318            "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
319            "seq": seq,
320            "elements": screen.elements,
321        })
322    }
323
324    pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
325        let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
326        let (_, screen) = self
327            .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
328            .await?;
329        let matches = sel.find(&screen);
330        let mut out = fields! {
331            "selector": sel.describe(),
332            "count": matches.len(),
333            "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
334        };
335        if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
336            out.insert(
337                "note".into(),
338                json!("this reading came without identifiers, so id= could not match; try again"),
339            );
340        }
341        Ok(out)
342    }
343
344    /// Read the screen after an input that began at `since`: wait `delay` for
345    /// the screen to react, then ask the phone for a new dump.
346    ///
347    /// Some apps make the phone send dumps unasked while they change
348    /// (Calculator on every key). Such a dump shows the screen as it was a
349    /// moment earlier and can arrive late enough to pass for the answer; the
350    /// calculator example then read "12×3" instead of "408". So with `careful`
351    /// (inputs that change the screen several times), or when the phone has
352    /// lately sent unasked dumps, the screen is read until two readings in a
353    /// row agree.
354    pub(crate) async fn observe_with(
355        &mut self,
356        since: Instant,
357        delay: Duration,
358        careful: bool,
359    ) -> Option<(u64, Screen)> {
360        tokio::time::sleep(delay).await;
361        let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
362        let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
363        let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
364        if !careful && !chatty {
365            return Some(latest);
366        }
367        for _ in 0..2 {
368            trace(|| "observe: reading again until two readings agree".into());
369            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
370                break;
371            };
372            let same = next.1.signature() == latest.1.signature();
373            latest = next;
374            if same {
375                break;
376            }
377        }
378        Some(latest)
379    }
380
381    pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
382        self.observe_with(since, delay, false).await
383    }
384
385    /// Read the screen after an input until it has changed from `before` and
386    /// then holds still, as a browser waits for a page to load. The phone
387    /// sends no event when a screen comes up, so "came up" is two readings in
388    /// a row that agree. A screen that does not change within `CHANGE_WAIT`,
389    /// or keeps moving until `SETTLE_WAIT`, comes back as it is then.
390    pub(crate) async fn observe_settled(
391        &mut self,
392        since: Instant,
393        before: &[ElementSignature],
394    ) -> Option<(u64, Screen)> {
395        let mut latest = self.observe(since, AFTER_TAP).await?;
396        while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
397            trace(|| "observe: the screen has not changed yet; reading again".into());
398            match self.screen(Fresh::Force, false, BuildOptions::default()).await {
399                Ok(next) => latest = next,
400                Err(_) => return Some(latest),
401            }
402        }
403        if latest.1.signature() == before {
404            return Some(latest);
405        }
406        while since.elapsed() < SETTLE_WAIT {
407            let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
408                break;
409            };
410            let same = next.1.signature() == latest.1.signature();
411            latest = next;
412            if same {
413                break;
414            }
415            trace(|| "observe: the screen is still moving; reading again".into());
416        }
417        Some(latest)
418    }
419
420    /// Read the screen and find where to tap for each target. Fails without
421    /// sending anything when a target is missing, ambiguous or covered.
422    async fn locate(
423        &mut self,
424        targets: &[Target],
425        fresh: Fresh,
426    ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
427        let need_ids = targets
428            .iter()
429            .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
430        let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
431        if screen.width <= 0.0 || screen.height <= 0.0 {
432            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
433        }
434        let mut points = Vec::with_capacity(targets.len());
435        for target in targets {
436            points.push(match target {
437                Target::Selector(sel) => {
438                    let el = resolve_one(sel, &screen)?;
439                    let (x, y) = tap_point(el, &screen)?;
440                    (x, y, Some(el.clone()))
441                }
442                Target::Point(x, y) => {
443                    if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
444                        return Err(Fail::invalid(format!(
445                            "point {x},{y} is outside the {}x{} screen",
446                            screen.width, screen.height
447                        )));
448                    }
449                    (*x, *y, screen.element_at(*x, *y).cloned())
450                }
451            });
452        }
453        Ok((screen, points))
454    }
455
456    /// Locate the targets on the latest reading when it is too old to reuse
457    /// as it is (an agent's turn takes seconds) but the phone confirms it:
458    /// every application still shows the same visible children (nothing came
459    /// up or went away) and every target is still where it was. Each question
460    /// takes tens of milliseconds; a full reading takes over a second. `None`
461    /// sends the caller to a full reading.
462    async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
463        let age = self.reading_age()?;
464        if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
465            return None;
466        }
467        let (_, dump, _) = self.ax.as_ref()?.latest()?;
468        let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
469        let ax = self.ax.as_mut()?;
470        for app in &dump.applications {
471            let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
472            let now: Vec<ObjKey> = ax
473                .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
474                .await?
475                .as_array()?
476                .iter()
477                .filter_map(ObjKey::from_ka)
478                .collect();
479            if now != was {
480                trace(|| "locate: the screen has changed since the last reading".into());
481                return None;
482            }
483        }
484        for (_, _, el) in &located.1 {
485            let el = el.as_ref()?;
486            let now = ax
487                .ask(el.key?, attrs::FRAME, ASK_WAIT)
488                .await?
489                .get("NS.rectval")
490                .and_then(|v| v.as_str())
491                .and_then(Rect::parse)?;
492            let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
493            if el.frame.as_ref().map(round) != Some(round(&now)) {
494                trace(|| "locate: a target has moved since the last reading".into());
495                return None;
496            }
497        }
498        trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
499        Some(located)
500    }
501
502    /// Locate the targets, make sure input can be sent, and locate them again
503    /// if getting ready took long enough for the screen to have moved on: a
504    /// cold input session takes seconds to open, and a tap must never go to
505    /// where something used to be.
506    async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
507        let located = match self.locate_confirmed(targets).await {
508            Some(l) => l,
509            None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
510        };
511        let started = std::time::Instant::now();
512        self.ensure_input().await?;
513        if started.elapsed() < Duration::from_millis(400) {
514            return Ok(located);
515        }
516        self.locate(targets, Fresh::Force).await
517    }
518
519    pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
520        if args.double && args.long_ms.is_some() {
521            return Err(Fail::invalid("--double and --long cannot be combined"));
522        }
523        if let Some(ms) = args.long_ms
524            && !(1..=60_000).contains(&ms)
525        {
526            return Err(Fail::invalid("--long must be 1..60000 ms"));
527        }
528        let target = parse_target(&args.target)?;
529        let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
530            Ok(located) => located,
531            Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
532        };
533        let (x, y, element) = points.remove(0);
534        let element_json = element.as_ref().map(brief);
535
536        let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
537        let before = screen.signature();
538        let input_began = Instant::now();
539        let gesture = if args.double {
540            Gesture::DoubleTap { x: nx, y: ny }
541        } else {
542            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
543        };
544        self.send_gesture(gesture).await?;
545        let after = if args.no_wait {
546            self.observe(input_began, AFTER_TAP).await
547        } else {
548            self.observe_settled(input_began, &before).await
549        };
550        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
551        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
552        if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
553            // Something of the system's is over the app now.
554            self.input_suspect = true;
555        }
556
557        let mut out = fields! {
558            "sent": true,
559            "verified": changed.is_some(),
560            "screen_changed": changed.unwrap_or(false),
561            "x": (x * 10.0).round() / 10.0,
562            "y": (y * 10.0).round() / 10.0,
563        };
564        if let Some(e) = element_json {
565            out.insert("element".into(), e);
566        }
567        if changed.is_none() {
568            out.insert(
569                "note".into(),
570                json!("the tap was sent but no reading came back to compare"),
571            );
572        }
573        if let Some(change) = app_change {
574            out.insert("app_changed".into(), json!(true));
575            out.insert("app".into(), change);
576        }
577        // The screen it waited for, so that no reading has to follow.
578        if !args.no_wait
579            && let Some((seq, screen)) = after {
580                out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
581                out.insert("apps".into(), json!(screen.apps));
582                out.insert("count".into(), json!(screen.elements.len()));
583                out.insert("seq".into(), json!(seq));
584                out.insert("elements".into(), json!(screen.elements));
585            }
586        Ok(out)
587    }
588
589    /// The last known screen size, when `read_failed` is the phone's
590    /// accessibility daemon giving no reading on an unlocked phone. Control
591    /// Center does that while a screen recording runs, until it is closed
592    /// (2026-11-12, over Wi-Fi and USB alike), and closing it takes a tap, a
593    /// swipe or a button: input that needs no element then goes out on this
594    /// size, unverified.
595    fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
596        if !daemon_silent(read_failed) {
597            return None;
598        }
599        self.screen_size
600    }
601
602    /// A tap on a point while the screen cannot be read (`unread_size`).
603    /// Nothing is read afterwards either: that would only wait out another
604    /// timeout.
605    async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
606        let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
607            return Err(read_failed);
608        };
609        let (x, y) = (*x, *y);
610        if x < 0.0 || y < 0.0 || x > w || y > h {
611            return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
612        }
613        let (nx, ny) = (normalize(x, w), normalize(y, h));
614        let gesture = if args.double {
615            Gesture::DoubleTap { x: nx, y: ny }
616        } else {
617            Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
618        };
619        self.send_gesture(gesture).await?;
620        Ok(fields! {
621            "sent": true,
622            "verified": false,
623            "screen_changed": false,
624            "x": (x * 10.0).round() / 10.0,
625            "y": (y * 10.0).round() / 10.0,
626            "note": UNREAD_NOTE,
627        })
628    }
629
630    /// Tap several targets in a row on the screen as it is now: every target
631    /// is found before the first tap goes
632    /// out, nothing is re-read in between, and the screen is read once at the
633    /// end. For keypads and other screens that hold still while tapped.
634    pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
635        if targets.is_empty() {
636            return Err(Fail::invalid("the tap list is empty"));
637        }
638        let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
639        let (screen, points) = self.locate_ready(&parsed).await?;
640        let before = screen.signature();
641        let normalized: Vec<(u16, u16)> = points
642            .iter()
643            .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
644            .collect();
645        let tapped: Vec<Value> = points
646            .iter()
647            .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
648            .collect();
649        let input_began = Instant::now();
650        for (i, (nx, ny)) in normalized.iter().enumerate() {
651            let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
652            if let Err(mut fail) = self.send_gesture(tap).await {
653                if i > 0 {
654                    // The earlier taps of the list did go out.
655                    fail.outcome = crate::reply::Outcome::Unknown;
656                }
657                return Err(fail
658                    .hint(t(
659                        "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
660                        "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
661                    ))
662                    .with("taps_sent", i));
663            }
664        }
665        let after = self.observe_with(input_began, AFTER_TAP, true).await;
666        let changed = after.as_ref().map(|(_, s)| s.signature() != before);
667        let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
668        let mut out = fields! {
669            "sent": true,
670            "verified": changed.is_some(),
671            "screen_changed": changed.unwrap_or(false),
672            "taps": tapped.len(),
673            "elements": tapped,
674        };
675        if let Some(change) = app_change {
676            out.insert("app_changed".into(), json!(true));
677            out.insert("app".into(), change);
678        }
679        Ok(out)
680    }
681
682    /// Drag between two points given in logical points and report whether the
683    /// screen changed.
684    pub(crate) async fn drag_points(
685        &mut self,
686        from: (f64, f64),
687        to: (f64, f64),
688        duration: Duration,
689        motion: Motion,
690    ) -> Res<(bool, Option<Screen>)> {
691        let (_, screen) = self
692            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
693            .await?;
694        if screen.width <= 0.0 || screen.height <= 0.0 {
695            return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
696        }
697        for (x, y) in [from, to] {
698            if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
699                return Err(Fail::invalid(format!(
700                    "point {x},{y} is outside the {}x{} screen",
701                    screen.width, screen.height
702                )));
703            }
704        }
705        let before = screen.signature();
706        let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
707        let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
708        let input_began = Instant::now();
709        self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
710        let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
711        if motion == Motion::Flick {
712            // The content coasts after a flick: read until two readings agree.
713            for _ in 0..3 {
714                let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
715                let still = match (&after, &next) {
716                    (Some(a), Some(b)) => a.signature() == b.signature(),
717                    _ => true,
718                };
719                after = next.or(after);
720                if still {
721                    break;
722                }
723            }
724        }
725        let changed = after.as_ref().is_some_and(|s| s.signature() != before);
726        Ok((changed, after))
727    }
728
729    async fn screen_size(&mut self) -> Res<(f64, f64)> {
730        let (_, screen) = self
731            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
732            .await?;
733        Ok((screen.width, screen.height))
734    }
735
736    pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
737        let read = self
738            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
739            .await;
740        let (w, h) = match &read {
741            Ok((_, screen)) => (screen.width, screen.height),
742            Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
743        };
744        let (from, to) = match (&args.direction, args.from, args.to) {
745            (Some(dir), None, None) => {
746                // The finger moves in the named direction across the middle.
747                let (cx, cy) = (w / 2.0, h / 2.0);
748                match dir.as_str() {
749                    "up" => ((cx, h * 0.70), (cx, h * 0.30)),
750                    "down" => ((cx, h * 0.30), (cx, h * 0.70)),
751                    "left" => ((w * 0.80, cy), (w * 0.20, cy)),
752                    "right" => ((w * 0.20, cy), (w * 0.80, cy)),
753                    other => {
754                        return Err(Fail::invalid(format!(
755                            "unknown direction `{other}` (up, down, left, right)"
756                        )));
757                    }
758                }
759            }
760            (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
761            _ => return Err(Fail::invalid("give a direction, or both from and to")),
762        };
763        let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
764        let motion = if args.drag { Motion::Place } else { Motion::Flick };
765        if read.is_err() {
766            // `unread_size`: drag on the last known size, read nothing after.
767            for (x, y) in [from, to] {
768                if x < 0.0 || y < 0.0 || x > w || y > h {
769                    return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
770                }
771            }
772            let a = (normalize(from.0, w), normalize(from.1, h));
773            let b = (normalize(to.0, w), normalize(to.1, h));
774            self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
775            return Ok(fields! {
776                "sent": true,
777                "verified": false,
778                "screen_changed": false,
779                "from": [from.0, from.1],
780                "to": [to.0, to.1],
781                "note": UNREAD_NOTE,
782            });
783        }
784        let (changed, after) = self.drag_points(from, to, duration, motion).await?;
785        Ok(fields! {
786            "sent": true,
787            "verified": after.is_some(),
788            "screen_changed": changed,
789            "from": [from.0, from.1],
790            "to": [to.0, to.1],
791        })
792    }
793
794    /// Move the content one step. "down" shows what is further down.
795    async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
796        let (w, h) = self.screen_size().await?;
797        let cx = w / 2.0;
798        let (from, to) = match direction {
799            "down" => ((cx, h * 0.68), (cx, h * 0.32)),
800            "up" => ((cx, h * 0.32), (cx, h * 0.68)),
801            other => {
802                return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
803            }
804        };
805        // A drag that slows to a stop moves the list by the finger's distance,
806        // without a fling that would overshoot.
807        self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
808            .await
809    }
810
811    pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
812        if args.direction != "up" && args.direction != "down" {
813            return Err(Fail::invalid("direction must be up or down"));
814        }
815        let Some(until) = &args.until else {
816            let (changed, after) = self.scroll_step(&args.direction).await?;
817            return Ok(fields! {
818                "sent": true,
819                "verified": after.is_some(),
820                "screen_changed": changed,
821                "scrolls": 1,
822            });
823        };
824        let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
825        let max = args.max.unwrap_or(10).clamp(1, 100);
826        let full = sel.needs_identifier();
827        let mut scrolls = 0u32;
828        let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
829        loop {
830            let matches = dedupe_targets(sel.find(&screen));
831            // Found means: on screen and tappable (not under a toolbar).
832            if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
833                if matches.len() > 1 {
834                    return Err(Fail::not_sent(
835                        "ambiguous_selector",
836                        format!("{} elements match {}", matches.len(), sel.describe()),
837                    )
838                    .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
839                    .with("scrolls", scrolls));
840                }
841                return Ok(fields! {
842                    "found": true,
843                    "scrolls": scrolls,
844                    "screen_changed": scrolls > 0,
845                    "element": brief(el),
846                });
847            }
848            if scrolls >= max {
849                return Err(Fail::new(
850                    crate::reply::Outcome::Done,
851                    "not_found",
852                    format!("{} not found after {scrolls} scrolls", sel.describe()),
853                )
854                .with("found", false)
855                .with("scrolls", scrolls)
856                .with("screen_summary", screen.summary(12)));
857            }
858            let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
859            scrolls += 1;
860            if !changed {
861                // One still screen can be a dropped drag; two in a row is the
862                // end of the list.
863                (changed, after) = self.scroll_step(&args.direction).await?;
864                scrolls += 1;
865            }
866            if !changed {
867                return Err(Fail::new(
868                    crate::reply::Outcome::Done,
869                    "not_found",
870                    format!(
871                        "{} not found; the list stopped moving after {scrolls} scrolls",
872                        sel.describe()
873                    ),
874                )
875                .with("found", false)
876                .with("scrolls", scrolls)
877                .with("screen_summary", screen.summary(12)));
878            }
879            screen = match (after, full) {
880                (Some(s), false) => s,
881                _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
882            };
883        }
884    }
885
886    pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
887        let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
888        let (_, screen) = self
889            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
890            .await?;
891        let before = Some(screen.signature());
892        let input_began = Instant::now();
893        let input = self.ensure_input().await?;
894        if let Err(e) = input.key(usage, &modifiers).await {
895            self.drop_input();
896            return Err(Fail::unknown("input_failed", format!("{e:#}")));
897        }
898        let changed = match before {
899            Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
900            None => None,
901        };
902        Ok(fields! {
903            "sent": true,
904            "verified": changed.is_some(),
905            "screen_changed": changed.unwrap_or(false),
906            "key": args.key,
907        })
908    }
909
910    pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
911        let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
912        // A button needs no screen size; the reading is only to compare with.
913        let before = match self
914            .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
915            .await
916        {
917            Ok((_, screen)) => Some(screen.signature()),
918            Err(read_failed) if daemon_silent(&read_failed) => None,
919            Err(read_failed) => return Err(read_failed),
920        };
921        let input_began = Instant::now();
922        self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
923        let changed = match &before {
924            Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
925            None => None,
926        };
927        let mut out = fields! {
928            "sent": true,
929            "verified": changed.is_some(),
930            "screen_changed": changed.unwrap_or(false),
931            "button": args.button,
932        };
933        if before.is_none() {
934            out.insert("note".into(), json!(UNREAD_NOTE));
935        }
936        Ok(out)
937    }
938
939    pub async fn cmd_apps(&mut self) -> Res {
940        let apps = self.list_apps(true).await?;
941        let list: Vec<Value> = apps
942            .iter()
943            .map(|a| {
944                json!({
945                    "bundle_id": a.bundle_identifier,
946                    "name": a.name,
947                    "version": a.version,
948                    "first_party": a.is_first_party,
949                    "removable": a.is_removable,
950                })
951            })
952            .collect();
953        Ok(fields! { "count": list.len(), "apps": list })
954    }
955
956    async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
957        // Something with a dot and no spaces is a bundle id; anything else is
958        // looked up by name.
959        if app.contains('.') && !app.contains(' ') {
960            return Ok(app.to_string());
961        }
962        let want = nfc(app).to_lowercase();
963        let apps = self.list_apps(false).await?;
964        let hits: Vec<&idevice::core_device::AppListEntry> = apps
965            .iter()
966            .filter(|a| nfc(&a.name).to_lowercase() == want)
967            .collect();
968        match hits.len() {
969            1 => Ok(hits[0].bundle_identifier.clone()),
970            0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
971                .hint(t(
972                    "`iph apps`로 이름과 bundle id를 확인하세요.",
973                    "See names and bundle ids with `iph apps`.",
974                ))),
975            _ => Err(Fail::not_sent(
976                "ambiguous_app",
977                format!("{} apps are named \"{app}\"", hits.len()),
978            )
979            .with(
980                "candidates",
981                Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
982            )),
983        }
984    }
985
986    pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
987        let bundle = self.resolve_bundle(&args.app).await?;
988        let mut svc = self.app_service().await?;
989        let launch_began = Instant::now();
990        // From here the launch request may have reached the phone.
991        let launched = tokio::time::timeout(
992            Duration::from_secs(30),
993            svc.launch_application(bundle.as_str(), &[], args.restart, false, None, None, None),
994        )
995        .await
996        .map_err(|_| "no answer within 30 s".to_string())
997        .and_then(|r| r.map_err(|e| format!("{e:?}")));
998        let resp = match launched {
999            Ok(r) => r,
1000            Err(msg) => {
1001                let locked = msg.to_lowercase().contains("lock");
1002                return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1003                    t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1004                } else {
1005                    t(
1006                        "bundle id가 맞는지 `iph apps`로 확인하세요.",
1007                        "Check the bundle id with `iph apps`.",
1008                    )
1009                }));
1010            }
1011        };
1012        let pid = resp.pid as i64;
1013        // The app is in front when a reading lists it as an application.
1014        let mut foreground = false;
1015        let mut delay = AFTER_LAUNCH;
1016        for _ in 0..4 {
1017            if let Some((_, screen)) = self.observe(launch_began, delay).await
1018                && screen.apps.iter().any(|a| a.pid == pid) {
1019                    foreground = true;
1020                    break;
1021                }
1022            delay = Duration::from_millis(300);
1023        }
1024        Ok(fields! {
1025            "sent": true,
1026            "verified": foreground,
1027            "foreground": foreground,
1028            "bundle_id": bundle,
1029            "pid": pid,
1030        })
1031    }
1032
1033    pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1034        let png = self.screenshot_png().await?;
1035        let path = std::path::PathBuf::from(&args.path);
1036        if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1037            std::fs::create_dir_all(dir)
1038                .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1039        }
1040        std::fs::write(&path, &png)
1041            .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1042        Ok(fields! { "path": args.path, "bytes": png.len() })
1043    }
1044
1045    /// The text element that has the keyboard, if exactly one does.
1046    fn focused_field(screen: &Screen) -> Option<&Element> {
1047        let mut focused = screen.elements.iter().filter(|e| e.focused);
1048        let first = focused.next()?;
1049        focused.next().is_none().then_some(first)
1050    }
1051
1052    /// What a field shows after a paste, asked of the phone directly: again
1053    /// every 0.1 s until it differs from before (the paste has landed) or
1054    /// `PASTE_SHOWS` has passed. `None` when the phone does not answer, and
1055    /// the caller reads the whole screen instead.
1056    async fn field_value(&mut self, field: &Element) -> Option<String> {
1057        let key = field.key?;
1058        let before = field.value.clone().unwrap_or_default();
1059        let began = Instant::now();
1060        loop {
1061            let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1062            let now = crate::ax::tree::text(Some(&value)).unwrap_or_default();
1063            if now != before || began.elapsed() >= PASTE_SHOWS {
1064                return Some(now);
1065            }
1066            tokio::time::sleep(Duration::from_millis(100)).await;
1067        }
1068    }
1069
1070    pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1071        if args.text.is_empty() && !args.clear {
1072            return Err(Fail::invalid("nothing to type"));
1073        }
1074        if args.text.len() > 1024 * 1024 {
1075            return Err(Fail::invalid("text is larger than 1 MiB"));
1076        }
1077        let (_, screen) = self.screen(Fresh::MaxAge(reuse()), false, BuildOptions::default()).await?;
1078        let Some(field) = Self::focused_field(&screen).cloned() else {
1079            return Err(Fail::not_sent(
1080                "no_focused_field",
1081                "no text field has the keyboard right now",
1082            )
1083            .hint(t(
1084                "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1085                "Tap the field first with `iph tap` so the keyboard comes up.",
1086            ))
1087            .with("screen_summary", screen.summary(12)));
1088        };
1089        trace(|| "type: focused field found".into());
1090
1091        // The input session first: if it cannot be had, nothing has been
1092        // touched yet, the clipboard included.
1093        self.ensure_input().await?;
1094        let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1095
1096        // Remember what the user had on the clipboard and, meanwhile, register
1097        // the temporary keyboard (no key goes out yet).
1098        let input = self.ensure_input().await?;
1099        let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1100        trace(|| "type: clipboard saved, keyboard attached".into());
1101        if let Err(e) = attached {
1102            // Nothing has been touched: the clipboard still holds the user's content.
1103            return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1104        }
1105        if !args.text.is_empty()
1106            && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1107                let _ = input.keyboard_detach().await;
1108                self.invalidate_readings();
1109                return Err(Fail::not_sent("clipboard_failed", e));
1110            }
1111        trace(|| "type: clipboard set".into());
1112
1113        let cmd = vec![crate::keys::MOD_COMMAND];
1114        let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1115        if args.clear {
1116            combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1117            combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1118        }
1119        if !args.text.is_empty() {
1120            combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1121        }
1122        let keys_began = Instant::now();
1123        let pressed = input.keyboard_press(&combos).await;
1124        trace(|| "type: keys sent".into());
1125        if let Err(e) = pressed {
1126            let _ = input.keyboard_detach().await;
1127            self.drop_input();
1128            self.invalidate_readings();
1129            let restored = saved.restore(&adapter, port, &args.text).await;
1130            return Err(Fail::unknown("input_failed", format!("{e:#}"))
1131                .hint(t(
1132                    "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1133                    "The input may have gone out. Do not resend; check the screen with `iph read`.",
1134                ))
1135                .with("clipboard_restored", restored));
1136        }
1137
1138        // The paste has happened by now, so the clipboard can go back while
1139        // the field is read. The temporary keyboard stays attached meanwhile:
1140        // the screen holds still and Return can follow at once. What the field
1141        // shows is reported, not judged: apps format, mask, correct and clear
1142        // what was pasted (₩ 30,000 for 30000), so only the caller can tell
1143        // whether it is right, as with Playwright's fill().
1144        let restoring = saved.restore(&adapter, port, &args.text);
1145        let reading = async {
1146            tokio::time::sleep(AFTER_PASTE).await;
1147            if let Some(value) = self.field_value(&field).await {
1148                return Some(value);
1149            }
1150            let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1151            Self::focused_field(&screen)
1152                .or_else(|| screen.by_id(&field.id))
1153                .map(|e| e.value.clone().unwrap_or_default())
1154        };
1155        let (restored, value) = tokio::join!(restoring, reading);
1156        trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1157
1158        let mut submitted = false;
1159        let mut submit_error = None;
1160        if args.submit
1161            && let Some(input) = self.input.as_mut() {
1162                match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1163                    Ok(()) => submitted = true,
1164                    Err(e) => submit_error = Some(format!("{e:#}")),
1165                }
1166            }
1167        if let Some(input) = self.input.as_mut()
1168            && input.keyboard_detach().await.is_err() {
1169                self.drop_input();
1170            }
1171        // The on-screen keyboard comes back now; readings from before show
1172        // the screen without it.
1173        self.invalidate_readings();
1174
1175        if let Some(e) = submit_error {
1176            self.drop_input();
1177            return Err(Fail::unknown("input_failed", e)
1178                .with("value", value.clone().map(Value::String).unwrap_or(Value::Null))
1179                .with("clipboard_restored", restored));
1180        }
1181        let mut out = fields! {
1182            "sent": true,
1183            "verified": value.is_some(),
1184            "value": value,
1185            "field": brief(&field),
1186            "submit_sent": submitted,
1187            "clipboard_restored": restored,
1188            "screen_changed": true,
1189        };
1190        if !restored && saved.unreadable() {
1191            out.insert(
1192                "clipboard_note".into(),
1193                json!(t(
1194                    "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1195                    "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1196                )),
1197            );
1198        }
1199        Ok(out)
1200    }
1201}
1202
1203const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1204
1205/// Kinds of data the phone's clipboard service holds itself: what `iph type`
1206/// puts there, and plain text or a picture. An item that carries any other kind
1207/// (rich text, an app's own format) was copied in an app, and its bytes are
1208/// only promised: the service asks that app for them when someone reads them.
1209/// Read from here, that request never came back. Text copied in Notes stalled
1210/// past 8 s for every kind on offer (rtf and plain text alike), and the
1211/// service then answered nothing at all, writes included, for about a minute
1212/// (2026-11-11: `iph type` right after ⌘C in Notes failed twice with "set
1213/// clipboard: no answer" and lost the clipboard the third time). Such content
1214/// is not read.
1215const SERVICE_KINDS: [&str; 6] = [
1216    "public.utf8-plain-text",
1217    "public.plain-text",
1218    "public.text",
1219    "public.png",
1220    "public.jpeg",
1221    "public.tiff",
1222];
1223/// What the service's own items took to read: 75 ~ 124 ms.
1224const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1225
1226/// Whether an item with these kinds was copied in an app, so reading its
1227/// bytes would wait on that app.
1228fn app_provided(types: &[String]) -> bool {
1229    types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1230}
1231
1232/// What the clipboard holds, as far as it can be read without waiting on an app.
1233#[derive(Debug, PartialEq)]
1234enum Peek {
1235    Empty,
1236    Text(String),
1237    Image(String, Vec<u8>),
1238    /// Copied in an app; left unread (see `SERVICE_KINDS`).
1239    AppContent,
1240    /// No answer, or something not understood.
1241    Unknown,
1242}
1243
1244/// What the user had on the phone's clipboard before `iph type` borrowed it.
1245struct Clipboard {
1246    before: Peek,
1247}
1248
1249impl Clipboard {
1250    /// Every call uses its own connection, so a late answer to one request
1251    /// cannot be taken for the answer to the next.
1252    async fn connect(
1253        adapter: &crate::tcp::handle::AdapterHandle,
1254        port: u16,
1255    ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1256        use idevice::RsdService;
1257        let mut adapter = adapter.clone();
1258        let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1259            .await
1260            .map_err(|_| "clipboard service: no answer".to_string())?
1261            .map_err(|e| format!("clipboard service: {e}"))?;
1262        idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1263            .await
1264            .map_err(|e| format!("clipboard service: {e:?}"))
1265    }
1266
1267    /// Look at the clipboard. Bytes come along only as promises, and only
1268    /// those the service holds itself are fetched.
1269    async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1270        use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1271        let general = idevice::core_device::GENERAL_PASTEBOARD;
1272        let Ok(mut clip) = Self::connect(adapter, port).await else {
1273            return Peek::Unknown;
1274        };
1275        let pulled = tokio::time::timeout(
1276            Duration::from_secs(8),
1277            clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1278        )
1279        .await;
1280        let Ok(Ok(snap)) = pulled else {
1281            return Peek::Unknown;
1282        };
1283        trace(|| {
1284            let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1285            format!("clipboard: kinds [{}]", kinds.join(" | "))
1286        });
1287        let Some(item) = snap.items.first() else {
1288            return Peek::Empty;
1289        };
1290        if snap.items.iter().any(|i| app_provided(&i.types)) {
1291            return Peek::AppContent;
1292        }
1293        // Text before a picture, in the order of `SERVICE_KINDS`.
1294        for kind in SERVICE_KINDS {
1295            let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1296                continue;
1297            };
1298            let bytes = match &entry.payload {
1299                PasteboardPayload::Inline(b) => Some(b.clone()),
1300                PasteboardPayload::Promised { .. } => {
1301                    match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1302                        Ok(Ok(b)) => b,
1303                        Ok(Err(_)) => None,
1304                        // Whatever holds it up may hold up the service too.
1305                        Err(_) => return Peek::Unknown,
1306                    }
1307                }
1308                PasteboardPayload::Error(_) => None,
1309            };
1310            let Some(bytes) = bytes else { continue };
1311            return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1312                Peek::Image(kind.to_string(), bytes)
1313            } else {
1314                match String::from_utf8(bytes) {
1315                    Ok(text) => Peek::Text(text),
1316                    Err(_) => continue,
1317                }
1318            };
1319        }
1320        Peek::Unknown
1321    }
1322
1323    async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1324        let mut clip = Self::connect(adapter, port).await?;
1325        let general = idevice::core_device::GENERAL_PASTEBOARD;
1326        match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1327            Ok(Ok(())) => Ok(()),
1328            Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1329            Err(_) => Err("set clipboard: no answer".into()),
1330        }
1331    }
1332
1333    async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1334        Clipboard { before: Self::peek(adapter, port).await }
1335    }
1336
1337    /// True when what was on the clipboard could not be read and is lost
1338    /// once `iph type` puts its text there.
1339    fn unreadable(&self) -> bool {
1340        matches!(self.before, Peek::AppContent | Peek::Unknown)
1341    }
1342
1343    /// Put the old content back. True when the clipboard is known to hold
1344    /// what it held before.
1345    async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1346        if typed.is_empty() {
1347            return true;
1348        }
1349        // Give the paste a moment to read the clipboard.
1350        tokio::time::sleep(Duration::from_millis(150)).await;
1351        // Only touch the clipboard if it still holds what we put there; the
1352        // user may have copied something else meanwhile.
1353        if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1354            return false;
1355        }
1356        let general = idevice::core_device::GENERAL_PASTEBOARD;
1357        let wrote = match &self.before {
1358            Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1359            Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1360                Ok(mut clip) => {
1361                    matches!(
1362                        tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1363                        Ok(Ok(()))
1364                    )
1365                }
1366                Err(_) => false,
1367            },
1368            // The clipboard was empty, or what was there is gone: either way
1369            // the typed text does not stay behind.
1370            Peek::Empty | Peek::AppContent | Peek::Unknown => {
1371                let _ = Self::set_text(adapter, port, "").await;
1372                return matches!(self.before, Peek::Empty);
1373            }
1374        };
1375        if !wrote {
1376            return false;
1377        }
1378        match (Self::peek(adapter, port).await, &self.before) {
1379            (Peek::Text(now), Peek::Text(old)) => &now == old,
1380            (Peek::Image(..), Peek::Image(..)) => true,
1381            _ => false,
1382        }
1383    }
1384}
1385
1386#[cfg(test)]
1387mod clipboard_tests {
1388    use super::app_provided;
1389
1390    fn kinds(k: &[&str]) -> Vec<String> {
1391        k.iter().map(|s| s.to_string()).collect()
1392    }
1393
1394    #[test]
1395    fn notes_copy_is_app_content() {
1396        // What ⌘C in Notes put on the clipboard (2026-11-11).
1397        assert!(app_provided(&kinds(&[
1398            "public.utf8-plain-text",
1399            "com.apple.notes.richtext",
1400            "com.apple.webarchive",
1401            "com.apple.flat-rtfd",
1402            "iOS rich content paste pasteboard type",
1403            "public.html",
1404            "net.daringfireball.markdown",
1405            "public.rtf",
1406        ])));
1407    }
1408
1409    #[test]
1410    fn what_iph_puts_there_is_not() {
1411        // `set_text` writes these three kinds.
1412        assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1413        assert!(!app_provided(&kinds(&["public.png"])));
1414    }
1415}