1use std::time::{Duration, Instant};
5
6use serde::Deserialize;
7use serde_json::{Value, json};
8use unicode_normalization::UnicodeNormalization;
9
10use crate::{
11 ax::{
12 attrs,
13 dump::ObjKey,
14 tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
15 },
16 device::{
17 ax::trace,
18 input::{Motion, normalize},
19 session::{Fresh, Gesture, Session, reuse},
20 },
21 fields,
22 i18n::t,
23 reply::{Fail, Res},
24 selector::{Selector, dedupe_targets},
25};
26
27pub const AFTER_TAP: Duration = Duration::from_millis(480);
31pub const AFTER_DRAG: Duration = Duration::from_millis(300);
33pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
34pub const AFTER_PASTE: Duration = Duration::from_millis(200);
35pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
39pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
41const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
45const PASTE_SHOWS: Duration = Duration::from_millis(1500);
47const ASK_WAIT: Duration = Duration::from_millis(1000);
49const UNASKED_MEMORY: Duration = Duration::from_secs(5);
52const UNREAD_NOTE: &str =
54 "the screen could not be read; sent on the last known screen size, not verified";
55
56#[derive(Debug, Default, Deserialize)]
57#[serde(default, deny_unknown_fields)]
58pub struct ReadArgs {
59 pub app: bool,
61 pub all: bool,
63 pub full: bool,
65 pub raw: bool,
67 pub max_age_ms: Option<u64>,
69}
70
71#[derive(Debug, Deserialize)]
72#[serde(deny_unknown_fields)]
73pub struct FindArgs {
74 pub selector: Value,
75 #[serde(default)]
76 pub all: bool,
77}
78
79#[derive(Debug, Default, Deserialize)]
80#[serde(default, deny_unknown_fields)]
81pub struct TapArgs {
82 pub target: Value,
84 pub long_ms: Option<u64>,
85 pub double: bool,
86 pub no_wait: bool,
89}
90
91#[derive(Debug, Default, Deserialize)]
92#[serde(default, deny_unknown_fields)]
93pub struct SwipeArgs {
94 pub direction: Option<String>,
95 pub from: Option<[f64; 2]>,
96 pub to: Option<[f64; 2]>,
97 pub duration_ms: Option<u64>,
98 pub drag: bool,
100}
101
102#[derive(Debug, Default, Deserialize)]
103#[serde(default, deny_unknown_fields)]
104pub struct ScrollArgs {
105 pub direction: String,
106 pub until: Option<Value>,
107 pub max: Option<u32>,
108}
109
110#[derive(Debug, Default, Deserialize)]
111#[serde(default, deny_unknown_fields)]
112pub struct TypeArgs {
113 pub text: String,
114 pub clear: bool,
115 pub submit: bool,
116}
117
118struct TypeObservation {
121 before: Option<String>,
122 value: Option<String>,
123 changed: Option<bool>,
124 warning: Option<Value>,
125}
126
127impl TypeObservation {
128 fn new(args: &TypeArgs, before: Option<String>, value: Option<String>) -> Self {
129 let changed = before.as_ref().zip(value.as_ref()).map(|(a, b)| a != b);
130 let retry_hint = t(
131 "입력이 반영되었을 수 있습니다. 자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요.",
132 "Input may have taken effect. Do not automatically retype; check with `iph read` or a screenshot.",
133 );
134 let warning = if value.is_none() {
135 Some(json!({
136 "code": "value_unreadable",
137 "message": "the input was sent but the field's value could not be read afterwards",
138 "hint": retry_hint,
139 }))
140 } else if args.text.is_empty() && value.as_deref() == Some("") {
141 None
143 } else if args.text.is_empty() {
144 Some(json!({
145 "code": "clear_not_observed",
146 "message": "the clear input was sent but the field still shows a nonempty value",
147 "hint": retry_hint,
148 }))
149 } else if changed == Some(false) {
150 Some(json!({
151 "code": "value_unchanged",
152 "message": "the field's value is unchanged; the paste may not have taken effect (identical replacement, masking or app behavior can also cause this)",
153 "hint": t(
154 "자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요. 붙여넣기가 막혔을 수도 있습니다: 설정 → 앱 → 해당 앱 → 다른 앱에서 붙여넣기 → 허용. 값이 같다는 사실만으로 권한 거부를 확정할 수는 없습니다.",
155 "Do not automatically retype; check with `iph read` or a screenshot. Pasting may be blocked: Settings → Apps → this app → Paste from Other Apps → Allow. An unchanged value alone does not prove permission was denied.",
156 ),
157 }))
158 } else if value.as_deref() == Some("") {
159 Some(json!({
160 "code": "value_empty",
161 "message": "the input was sent but the field shows an empty value; the app may have cleared it",
162 "hint": retry_hint,
163 }))
164 } else if changed.is_none() {
165 Some(json!({
166 "code": "value_change_unknown",
167 "message": "the field's value was read afterwards, but its previous value was unavailable for comparison",
168 "hint": retry_hint,
169 }))
170 } else {
171 None
172 };
173 Self {
174 before,
175 value,
176 changed,
177 warning,
178 }
179 }
180
181 fn can_submit(&self) -> bool {
182 self.warning.is_none()
183 }
184
185 fn result(
186 self,
187 args: &TypeArgs,
188 field: &Element,
189 restored: bool,
190 submitted: Option<bool>,
191 screen_changed: Option<bool>,
192 submit_error: Option<String>,
193 ) -> Res {
194 let blocked_submit = args.submit && !self.can_submit();
195 let mut out = fields! {
196 "sent": true,
197 "verified": self.value.is_some(),
198 "value_before": self.before,
199 "value": self.value,
200 "value_changed": self.changed,
201 "field": brief(field),
202 "submit_sent": submitted,
203 "clipboard_restored": restored,
204 "screen_changed": screen_changed,
205 };
206 if let Some(warning) = self.warning {
207 out.insert("warning".into(), warning);
208 }
209 let fail = if let Some(e) = submit_error {
210 Some(Fail::unknown("input_failed", e))
211 } else if blocked_submit {
212 Some(Fail::unknown(
213 "type_not_verified",
214 "the input was sent, but its effect is uncertain; Return was not sent",
215 ))
216 } else {
217 None
218 };
219 if let Some(mut fail) = fail {
220 fail.hint = Some(t(
221 "입력은 이미 나갔습니다. 자동으로 재입력하거나 제출하지 말고 `iph read`나 스크린샷으로 먼저 확인하세요.",
222 "Input has already been sent. Do not automatically retype or submit; check with `iph read` or a screenshot first.",
223 ).into());
224 fail.extra = out;
225 Err(fail)
226 } else {
227 Ok(out)
228 }
229 }
230}
231
232#[derive(Debug, Deserialize)]
233#[serde(deny_unknown_fields)]
234pub struct KeyArgs {
235 pub key: String,
236}
237
238#[derive(Debug, Deserialize)]
239#[serde(deny_unknown_fields)]
240pub struct ButtonArgs {
241 pub button: String,
242}
243
244#[derive(Debug, Default, Deserialize)]
245#[serde(deny_unknown_fields)]
246pub struct LaunchArgs {
247 pub app: String,
248 #[serde(default)]
250 pub restart: bool,
251}
252
253#[derive(Debug, Deserialize)]
254#[serde(deny_unknown_fields)]
255pub struct ScreenshotArgs {
256 pub path: String,
257}
258
259pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
260 let v = if args.is_null() { json!({}) } else { args.clone() };
261 serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
262}
263
264pub fn brief(el: &Element) -> Value {
265 let mut v = json!({ "id": el.id, "role": el.role });
266 if let Some(l) = &el.label {
267 v["label"] = json!(l);
268 }
269 if let Some(val) = &el.value {
270 v["value"] = json!(val);
271 }
272 if let Some(i) = &el.identifier {
273 v["identifier"] = json!(i);
274 }
275 if let Some(f) = &el.frame {
276 v["frame"] = json!(f);
277 }
278 v
279}
280
281pub enum Target {
282 Selector(Selector),
283 Point(f64, f64),
284}
285
286fn daemon_silent(read_failed: &Fail) -> bool {
289 let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
290 read_failed.code == "ax_timeout" && !locked
291}
292
293pub fn parse_target(v: &Value) -> Res<Target> {
294 if let Some(obj) = v.as_object()
295 && let (Some(x), Some(y)) = (
296 obj.get("x").and_then(|n| n.as_f64()),
297 obj.get("y").and_then(|n| n.as_f64()),
298 )
299 && obj.len() == 2 {
300 return Ok(Target::Point(x, y));
301 }
302 if let Some(s) = v.as_str() {
303 let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
304 if parts.len() == 2
305 && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
306 return Ok(Target::Point(x, y));
307 }
308 }
309 Selector::from_value(v)
310 .map(Target::Selector)
311 .map_err(|e| Fail::invalid(e.to_string()))
312}
313
314pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
316 let matches = dedupe_targets(sel.find(screen));
317 match matches.len() {
318 1 => Ok(matches[0]),
319 0 => Err(Fail::not_sent(
320 "not_found",
321 format!("no element matches {}", sel.describe()),
322 )
323 .hint(t(
324 "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
325 "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
326 ))
327 .with("selector", sel.describe())
328 .with("screen_summary", screen.summary(12))),
329 n => Err(Fail::not_sent(
330 "ambiguous_selector",
331 format!("{n} elements match {}", sel.describe()),
332 )
333 .hint(t(
334 "role=, within=, index= 으로 하나만 남게 좁히세요.",
335 "Narrow it to one with role=, within= or index=.",
336 ))
337 .with("selector", sel.describe())
338 .with(
339 "candidates",
340 Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
341 )),
342 }
343}
344
345fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
347 let frame = el.frame.ok_or_else(|| {
348 Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
349 })?;
350 let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
351 let visible = frame.intersection(&bounds).ok_or_else(|| {
352 Fail::not_sent("off_screen", "the element is outside the screen")
353 .hint(t(
354 "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
355 "Bring it on screen with `iph scroll down --until …`.",
356 ))
357 .with("element", brief(el))
358 })?;
359 let covered = |x: f64, y: f64| -> Option<&Element> {
360 let hit = screen.element_at(x, y)?;
361 if hit.id == el.id {
362 return None;
363 }
364 let inside = hit.frame.is_some_and(|h| {
367 h.x >= frame.x - 1.0
368 && h.y >= frame.y - 1.0
369 && h.x + h.w <= frame.x + frame.w + 1.0
370 && h.y + h.h <= frame.y + frame.h + 1.0
371 });
372 let encloses = hit.frame.is_some_and(|h| {
373 frame.x >= h.x - 1.0
374 && frame.y >= h.y - 1.0
375 && frame.x + frame.w <= h.x + h.w + 1.0
376 && frame.y + frame.h <= h.y + h.h + 1.0
377 });
378 (!inside && !encloses).then_some(hit)
379 };
380 let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
381 let mut blocker = None;
382 for (fx, fy) in candidates {
383 let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
384 match covered(x, y) {
385 None => return Ok((x, y)),
386 Some(b) => blocker = blocker.or(Some(b)),
387 }
388 }
389 Err(Fail::not_sent("covered", "another element covers the target")
390 .hint(t(
391 "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
392 "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
393 ))
394 .with("element", brief(el))
395 .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
396}
397
398fn nfc(s: &str) -> String {
399 s.nfc().collect()
400}
401
402fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
404 let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
405 let now = after.apps.first()?;
406 (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
407}
408
409impl Session {
410 fn build_opts(all: bool, raw: bool) -> BuildOptions {
411 BuildOptions { include_offscreen: all, raw }
412 }
413
414 pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
415 let fresh = match args.max_age_ms {
416 Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
417 None => Fresh::Force,
418 };
419 let (seq, mut screen) = self
420 .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
421 .await?;
422 if args.app
423 && let Some(front) = screen.apps.first().map(|a| a.pid) {
424 screen.elements.retain(|e| e.pid == front);
425 screen.apps.retain(|a| a.pid == front);
426 }
427 let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
428 Ok(fields! {
429 "screen": {"width": screen.width, "height": screen.height},
430 "apps": screen.apps,
431 "count": screen.elements.len(),
432 "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
433 "seq": seq,
434 "elements": screen.elements,
435 })
436 }
437
438 pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
439 let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
440 let (_, screen) = self
441 .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
442 .await?;
443 let matches = sel.find(&screen);
444 let mut out = fields! {
445 "selector": sel.describe(),
446 "count": matches.len(),
447 "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
448 };
449 if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
450 out.insert(
451 "note".into(),
452 json!("this reading came without identifiers, so id= could not match; try again"),
453 );
454 }
455 Ok(out)
456 }
457
458 pub(crate) async fn observe_with(
469 &mut self,
470 since: Instant,
471 delay: Duration,
472 careful: bool,
473 ) -> Option<(u64, Screen)> {
474 tokio::time::sleep(delay).await;
475 let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
476 let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
477 let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
478 if !careful && !chatty {
479 return Some(latest);
480 }
481 for _ in 0..2 {
482 trace(|| "observe: reading again until two readings agree".into());
483 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
484 break;
485 };
486 let same = next.1.signature() == latest.1.signature();
487 latest = next;
488 if same {
489 break;
490 }
491 }
492 Some(latest)
493 }
494
495 pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
496 self.observe_with(since, delay, false).await
497 }
498
499 pub(crate) async fn observe_settled(
505 &mut self,
506 since: Instant,
507 before: &[ElementSignature],
508 ) -> Option<(u64, Screen)> {
509 let mut latest = self.observe(since, AFTER_TAP).await?;
510 while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
511 trace(|| "observe: the screen has not changed yet; reading again".into());
512 match self.screen(Fresh::Force, false, BuildOptions::default()).await {
513 Ok(next) => latest = next,
514 Err(_) => return Some(latest),
515 }
516 }
517 if latest.1.signature() == before {
518 return Some(latest);
519 }
520 while since.elapsed() < SETTLE_WAIT {
521 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
522 break;
523 };
524 let same = next.1.signature() == latest.1.signature();
525 latest = next;
526 if same {
527 break;
528 }
529 trace(|| "observe: the screen is still moving; reading again".into());
530 }
531 Some(latest)
532 }
533
534 async fn locate(
537 &mut self,
538 targets: &[Target],
539 fresh: Fresh,
540 ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
541 let need_ids = targets
542 .iter()
543 .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
544 let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
545 if screen.width <= 0.0 || screen.height <= 0.0 {
546 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
547 }
548 let mut points = Vec::with_capacity(targets.len());
549 for target in targets {
550 points.push(match target {
551 Target::Selector(sel) => {
552 let el = resolve_one(sel, &screen)?;
553 let (x, y) = tap_point(el, &screen)?;
554 (x, y, Some(el.clone()))
555 }
556 Target::Point(x, y) => {
557 if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
558 return Err(Fail::invalid(format!(
559 "point {x},{y} is outside the {}x{} screen",
560 screen.width, screen.height
561 )));
562 }
563 (*x, *y, screen.element_at(*x, *y).cloned())
564 }
565 });
566 }
567 Ok((screen, points))
568 }
569
570 async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
577 let age = self.reading_age()?;
578 if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
579 return None;
580 }
581 let (_, dump, _) = self.ax.as_ref()?.latest()?;
582 let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
583 let ax = self.ax.as_mut()?;
584 for app in &dump.applications {
585 let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
586 let now: Vec<ObjKey> = ax
587 .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
588 .await?
589 .as_array()?
590 .iter()
591 .filter_map(ObjKey::from_ka)
592 .collect();
593 if now != was {
594 trace(|| "locate: the screen has changed since the last reading".into());
595 return None;
596 }
597 }
598 for (_, _, el) in &located.1 {
599 let el = el.as_ref()?;
600 let now = ax
601 .ask(el.key?, attrs::FRAME, ASK_WAIT)
602 .await?
603 .get("NS.rectval")
604 .and_then(|v| v.as_str())
605 .and_then(Rect::parse)?;
606 let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
607 if el.frame.as_ref().map(round) != Some(round(&now)) {
608 trace(|| "locate: a target has moved since the last reading".into());
609 return None;
610 }
611 }
612 trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
613 Some(located)
614 }
615
616 async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
621 let located = match self.locate_confirmed(targets).await {
622 Some(l) => l,
623 None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
624 };
625 let started = std::time::Instant::now();
626 self.ensure_input().await?;
627 if started.elapsed() < Duration::from_millis(400) {
628 return Ok(located);
629 }
630 self.locate(targets, Fresh::Force).await
631 }
632
633 pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
634 if args.double && args.long_ms.is_some() {
635 return Err(Fail::invalid("--double and --long cannot be combined"));
636 }
637 if let Some(ms) = args.long_ms
638 && !(1..=60_000).contains(&ms)
639 {
640 return Err(Fail::invalid("--long must be 1..60000 ms"));
641 }
642 let target = parse_target(&args.target)?;
643 let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
644 Ok(located) => located,
645 Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
646 };
647 let (x, y, element) = points.remove(0);
648 let element_json = element.as_ref().map(brief);
649
650 let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
651 let before = screen.signature();
652 let input_began = Instant::now();
653 let gesture = if args.double {
654 Gesture::DoubleTap { x: nx, y: ny }
655 } else {
656 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
657 };
658 self.send_gesture(gesture).await?;
659 let after = if args.no_wait {
660 self.observe(input_began, AFTER_TAP).await
661 } else {
662 self.observe_settled(input_began, &before).await
663 };
664 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
665 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
666 if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
667 self.input_suspect = true;
669 }
670
671 let mut out = fields! {
672 "sent": true,
673 "verified": changed.is_some(),
674 "screen_changed": changed.unwrap_or(false),
675 "x": (x * 10.0).round() / 10.0,
676 "y": (y * 10.0).round() / 10.0,
677 };
678 if let Some(e) = element_json {
679 out.insert("element".into(), e);
680 }
681 if changed.is_none() {
682 out.insert(
683 "note".into(),
684 json!("the tap was sent but no reading came back to compare"),
685 );
686 }
687 if let Some(change) = app_change {
688 out.insert("app_changed".into(), json!(true));
689 out.insert("app".into(), change);
690 }
691 if !args.no_wait
693 && let Some((seq, screen)) = after {
694 out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
695 out.insert("apps".into(), json!(screen.apps));
696 out.insert("count".into(), json!(screen.elements.len()));
697 out.insert("seq".into(), json!(seq));
698 out.insert("elements".into(), json!(screen.elements));
699 }
700 Ok(out)
701 }
702
703 fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
710 if !daemon_silent(read_failed) {
711 return None;
712 }
713 self.screen_size
714 }
715
716 async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
720 let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
721 return Err(read_failed);
722 };
723 let (x, y) = (*x, *y);
724 if x < 0.0 || y < 0.0 || x > w || y > h {
725 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
726 }
727 let (nx, ny) = (normalize(x, w), normalize(y, h));
728 let gesture = if args.double {
729 Gesture::DoubleTap { x: nx, y: ny }
730 } else {
731 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
732 };
733 self.send_gesture(gesture).await?;
734 Ok(fields! {
735 "sent": true,
736 "verified": false,
737 "screen_changed": false,
738 "x": (x * 10.0).round() / 10.0,
739 "y": (y * 10.0).round() / 10.0,
740 "note": UNREAD_NOTE,
741 })
742 }
743
744 pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
749 if targets.is_empty() {
750 return Err(Fail::invalid("the tap list is empty"));
751 }
752 let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
753 let (screen, points) = self.locate_ready(&parsed).await?;
754 let before = screen.signature();
755 let normalized: Vec<(u16, u16)> = points
756 .iter()
757 .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
758 .collect();
759 let tapped: Vec<Value> = points
760 .iter()
761 .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
762 .collect();
763 let input_began = Instant::now();
764 for (i, (nx, ny)) in normalized.iter().enumerate() {
765 let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
766 if let Err(mut fail) = self.send_gesture(tap).await {
767 if i > 0 {
768 fail.outcome = crate::reply::Outcome::Unknown;
770 }
771 return Err(fail
772 .hint(t(
773 "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
774 "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
775 ))
776 .with("taps_sent", i));
777 }
778 }
779 let after = self.observe_with(input_began, AFTER_TAP, true).await;
780 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
781 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
782 let mut out = fields! {
783 "sent": true,
784 "verified": changed.is_some(),
785 "screen_changed": changed.unwrap_or(false),
786 "taps": tapped.len(),
787 "elements": tapped,
788 };
789 if let Some(change) = app_change {
790 out.insert("app_changed".into(), json!(true));
791 out.insert("app".into(), change);
792 }
793 Ok(out)
794 }
795
796 pub(crate) async fn drag_points(
799 &mut self,
800 from: (f64, f64),
801 to: (f64, f64),
802 duration: Duration,
803 motion: Motion,
804 ) -> Res<(bool, Option<Screen>)> {
805 let (_, screen) = self
806 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
807 .await?;
808 if screen.width <= 0.0 || screen.height <= 0.0 {
809 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
810 }
811 for (x, y) in [from, to] {
812 if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
813 return Err(Fail::invalid(format!(
814 "point {x},{y} is outside the {}x{} screen",
815 screen.width, screen.height
816 )));
817 }
818 }
819 let before = screen.signature();
820 let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
821 let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
822 let input_began = Instant::now();
823 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
824 let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
825 if motion == Motion::Flick {
826 for _ in 0..3 {
828 let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
829 let still = match (&after, &next) {
830 (Some(a), Some(b)) => a.signature() == b.signature(),
831 _ => true,
832 };
833 after = next.or(after);
834 if still {
835 break;
836 }
837 }
838 }
839 let changed = after.as_ref().is_some_and(|s| s.signature() != before);
840 Ok((changed, after))
841 }
842
843 async fn screen_size(&mut self) -> Res<(f64, f64)> {
844 let (_, screen) = self
845 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
846 .await?;
847 Ok((screen.width, screen.height))
848 }
849
850 pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
851 let read = self
852 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
853 .await;
854 let (w, h) = match &read {
855 Ok((_, screen)) => (screen.width, screen.height),
856 Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
857 };
858 let (from, to) = match (&args.direction, args.from, args.to) {
859 (Some(dir), None, None) => {
860 let (cx, cy) = (w / 2.0, h / 2.0);
862 match dir.as_str() {
863 "up" => ((cx, h * 0.70), (cx, h * 0.30)),
864 "down" => ((cx, h * 0.30), (cx, h * 0.70)),
865 "left" => ((w * 0.80, cy), (w * 0.20, cy)),
866 "right" => ((w * 0.20, cy), (w * 0.80, cy)),
867 other => {
868 return Err(Fail::invalid(format!(
869 "unknown direction `{other}` (up, down, left, right)"
870 )));
871 }
872 }
873 }
874 (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
875 _ => return Err(Fail::invalid("give a direction, or both from and to")),
876 };
877 let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
878 let motion = if args.drag { Motion::Place } else { Motion::Flick };
879 if read.is_err() {
880 for (x, y) in [from, to] {
882 if x < 0.0 || y < 0.0 || x > w || y > h {
883 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
884 }
885 }
886 let a = (normalize(from.0, w), normalize(from.1, h));
887 let b = (normalize(to.0, w), normalize(to.1, h));
888 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
889 return Ok(fields! {
890 "sent": true,
891 "verified": false,
892 "screen_changed": false,
893 "from": [from.0, from.1],
894 "to": [to.0, to.1],
895 "note": UNREAD_NOTE,
896 });
897 }
898 let (changed, after) = self.drag_points(from, to, duration, motion).await?;
899 Ok(fields! {
900 "sent": true,
901 "verified": after.is_some(),
902 "screen_changed": changed,
903 "from": [from.0, from.1],
904 "to": [to.0, to.1],
905 })
906 }
907
908 async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
910 let (w, h) = self.screen_size().await?;
911 let cx = w / 2.0;
912 let (from, to) = match direction {
913 "down" => ((cx, h * 0.68), (cx, h * 0.32)),
914 "up" => ((cx, h * 0.32), (cx, h * 0.68)),
915 other => {
916 return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
917 }
918 };
919 self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
922 .await
923 }
924
925 pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
926 if args.direction != "up" && args.direction != "down" {
927 return Err(Fail::invalid("direction must be up or down"));
928 }
929 let Some(until) = &args.until else {
930 let (changed, after) = self.scroll_step(&args.direction).await?;
931 return Ok(fields! {
932 "sent": true,
933 "verified": after.is_some(),
934 "screen_changed": changed,
935 "scrolls": 1,
936 });
937 };
938 let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
939 let max = args.max.unwrap_or(10).clamp(1, 100);
940 let full = sel.needs_identifier();
941 let mut scrolls = 0u32;
942 let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
943 loop {
944 let matches = dedupe_targets(sel.find(&screen));
945 if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
947 if matches.len() > 1 {
948 return Err(Fail::not_sent(
949 "ambiguous_selector",
950 format!("{} elements match {}", matches.len(), sel.describe()),
951 )
952 .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
953 .with("scrolls", scrolls));
954 }
955 return Ok(fields! {
956 "found": true,
957 "scrolls": scrolls,
958 "screen_changed": scrolls > 0,
959 "element": brief(el),
960 });
961 }
962 if scrolls >= max {
963 return Err(Fail::new(
964 crate::reply::Outcome::Done,
965 "not_found",
966 format!("{} not found after {scrolls} scrolls", sel.describe()),
967 )
968 .with("found", false)
969 .with("scrolls", scrolls)
970 .with("screen_summary", screen.summary(12)));
971 }
972 let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
973 scrolls += 1;
974 if !changed {
975 (changed, after) = self.scroll_step(&args.direction).await?;
978 scrolls += 1;
979 }
980 if !changed {
981 return Err(Fail::new(
982 crate::reply::Outcome::Done,
983 "not_found",
984 format!(
985 "{} not found; the list stopped moving after {scrolls} scrolls",
986 sel.describe()
987 ),
988 )
989 .with("found", false)
990 .with("scrolls", scrolls)
991 .with("screen_summary", screen.summary(12)));
992 }
993 screen = match (after, full) {
994 (Some(s), false) => s,
995 _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
996 };
997 }
998 }
999
1000 pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
1001 let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
1002 let (_, screen) = self
1003 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1004 .await?;
1005 let before = Some(screen.signature());
1006 let input_began = Instant::now();
1007 let input = self.ensure_input().await?;
1008 if let Err(e) = input.key(usage, &modifiers).await {
1009 self.drop_input();
1010 return Err(Fail::unknown("input_failed", format!("{e:#}")));
1011 }
1012 let changed = match before {
1013 Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
1014 None => None,
1015 };
1016 Ok(fields! {
1017 "sent": true,
1018 "verified": changed.is_some(),
1019 "screen_changed": changed.unwrap_or(false),
1020 "key": args.key,
1021 })
1022 }
1023
1024 pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
1025 let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
1026 let before = match self
1028 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1029 .await
1030 {
1031 Ok((_, screen)) => Some(screen.signature()),
1032 Err(read_failed) if daemon_silent(&read_failed) => None,
1033 Err(read_failed) => return Err(read_failed),
1034 };
1035 let input_began = Instant::now();
1036 self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
1037 let changed = match &before {
1038 Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
1039 None => None,
1040 };
1041 let mut out = fields! {
1042 "sent": true,
1043 "verified": changed.is_some(),
1044 "screen_changed": changed.unwrap_or(false),
1045 "button": args.button,
1046 };
1047 if before.is_none() {
1048 out.insert("note".into(), json!(UNREAD_NOTE));
1049 }
1050 Ok(out)
1051 }
1052
1053 pub async fn cmd_apps(&mut self) -> Res {
1054 let apps = self.list_apps(true).await?;
1055 let list: Vec<Value> = apps
1056 .iter()
1057 .map(|a| {
1058 json!({
1059 "bundle_id": a.bundle_identifier,
1060 "name": a.name,
1061 "version": a.version,
1062 "first_party": a.is_first_party,
1063 "removable": a.is_removable,
1064 })
1065 })
1066 .collect();
1067 Ok(fields! { "count": list.len(), "apps": list })
1068 }
1069
1070 async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
1071 if app.contains('.') && !app.contains(' ') {
1074 return Ok(app.to_string());
1075 }
1076 let want = nfc(app).to_lowercase();
1077 let apps = self.list_apps(false).await?;
1078 let hits: Vec<&idevice::core_device::AppListEntry> = apps
1079 .iter()
1080 .filter(|a| nfc(&a.name).to_lowercase() == want)
1081 .collect();
1082 match hits.len() {
1083 1 => Ok(hits[0].bundle_identifier.clone()),
1084 0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
1085 .hint(t(
1086 "`iph apps`로 이름과 bundle id를 확인하세요.",
1087 "See names and bundle ids with `iph apps`.",
1088 ))),
1089 _ => Err(Fail::not_sent(
1090 "ambiguous_app",
1091 format!("{} apps are named \"{app}\"", hits.len()),
1092 )
1093 .with(
1094 "candidates",
1095 Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
1096 )),
1097 }
1098 }
1099
1100 pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
1101 let bundle = self.resolve_bundle(&args.app).await?;
1102 let mut svc = self.app_service().await?;
1103 let launch_began = Instant::now();
1104 let launched = tokio::time::timeout(
1106 Duration::from_secs(30),
1107 svc.launch_application(bundle.as_str(), &[], args.restart, false, None, None, None),
1108 )
1109 .await
1110 .map_err(|_| "no answer within 30 s".to_string())
1111 .and_then(|r| r.map_err(|e| format!("{e:?}")));
1112 let resp = match launched {
1113 Ok(r) => r,
1114 Err(msg) => {
1115 let locked = msg.to_lowercase().contains("lock");
1116 return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1117 t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1118 } else {
1119 t(
1120 "bundle id가 맞는지 `iph apps`로 확인하세요.",
1121 "Check the bundle id with `iph apps`.",
1122 )
1123 }));
1124 }
1125 };
1126 let pid = resp.pid as i64;
1127 let mut foreground = false;
1129 let mut delay = AFTER_LAUNCH;
1130 for _ in 0..4 {
1131 if let Some((_, screen)) = self.observe(launch_began, delay).await
1132 && screen.apps.iter().any(|a| a.pid == pid) {
1133 foreground = true;
1134 break;
1135 }
1136 delay = Duration::from_millis(300);
1137 }
1138 Ok(fields! {
1139 "sent": true,
1140 "verified": foreground,
1141 "foreground": foreground,
1142 "bundle_id": bundle,
1143 "pid": pid,
1144 })
1145 }
1146
1147 pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1148 let png = self.screenshot_png().await?;
1149 let path = std::path::PathBuf::from(&args.path);
1150 if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1151 std::fs::create_dir_all(dir)
1152 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1153 }
1154 std::fs::write(&path, &png)
1155 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1156 Ok(fields! { "path": args.path, "bytes": png.len() })
1157 }
1158
1159 fn focused_field(screen: &Screen) -> Option<&Element> {
1161 let mut focused = screen.elements.iter().filter(|e| e.focused);
1162 let first = focused.next()?;
1163 focused.next().is_none().then_some(first)
1164 }
1165
1166 async fn field_value(&mut self, field: &Element) -> Option<String> {
1171 let key = field.key?;
1172 let before = field.value.as_deref();
1173 let began = Instant::now();
1174 loop {
1175 let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1176 let now = crate::ax::tree::value_text(Some(&value))?;
1177 if Some(now.as_str()) != before || began.elapsed() >= PASTE_SHOWS {
1178 return Some(now);
1179 }
1180 tokio::time::sleep(Duration::from_millis(100)).await;
1181 }
1182 }
1183
1184 pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1185 if args.text.is_empty() && !args.clear {
1186 return Err(Fail::invalid("nothing to type"));
1187 }
1188 if args.text.len() > 1024 * 1024 {
1189 return Err(Fail::invalid("text is larger than 1 MiB"));
1190 }
1191 let (_, screen) = self
1192 .screen(Fresh::Force, false, BuildOptions::default())
1193 .await?;
1194 let Some(field) = Self::focused_field(&screen).cloned() else {
1195 return Err(Fail::not_sent(
1196 "no_focused_field",
1197 "no text field has the keyboard right now",
1198 )
1199 .hint(t(
1200 "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1201 "Tap the field first with `iph tap` so the keyboard comes up.",
1202 ))
1203 .with("screen_summary", screen.summary(12)));
1204 };
1205 trace(|| "type: focused field found".into());
1206
1207 self.ensure_input().await?;
1210 let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1211
1212 let input = self.ensure_input().await?;
1215 let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1216 trace(|| "type: clipboard saved, keyboard attached".into());
1217 if let Err(e) = attached {
1218 return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1220 }
1221 if !args.text.is_empty()
1222 && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1223 let _ = input.keyboard_detach().await;
1224 self.invalidate_readings();
1225 return Err(Fail::not_sent("clipboard_failed", e));
1226 }
1227 trace(|| "type: clipboard set".into());
1228
1229 let cmd = vec![crate::keys::MOD_COMMAND];
1230 let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1231 if args.clear {
1232 combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1233 combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1234 }
1235 if !args.text.is_empty() {
1236 combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1237 }
1238 let keys_began = Instant::now();
1239 let pressed = input.keyboard_press(&combos).await;
1240 trace(|| "type: keys sent".into());
1241 if let Err(e) = pressed {
1242 let _ = input.keyboard_detach().await;
1243 self.drop_input();
1244 self.invalidate_readings();
1245 let restored = saved.restore(&adapter, port, &args.text).await;
1246 return Err(Fail::unknown("input_failed", format!("{e:#}"))
1247 .hint(t(
1248 "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1249 "The input may have gone out. Do not resend; check the screen with `iph read`.",
1250 ))
1251 .with("clipboard_restored", restored));
1252 }
1253
1254 let restoring = saved.restore(&adapter, port, &args.text);
1261 let reading = async {
1262 tokio::time::sleep(AFTER_PASTE).await;
1263 if let Some(value) = self.field_value(&field).await {
1264 return Some(value);
1265 }
1266 let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1267 screen.by_id(&field.id).and_then(|e| e.value.clone())
1270 };
1271 let (restored, value) = tokio::join!(restoring, reading);
1272 trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1273 let observation = TypeObservation::new(&args, field.value.clone(), value);
1274
1275 let mut submitted = Some(false);
1276 let mut submit_error = None;
1277 if args.submit && observation.can_submit() {
1278 if let Some(input) = self.input.as_mut() {
1279 match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1280 Ok(()) => submitted = Some(true),
1281 Err(e) => {
1282 submitted = None;
1283 submit_error = Some(format!("{e:#}"));
1284 }
1285 }
1286 } else {
1287 submit_error = Some("the input session closed before Return could be sent".into());
1288 }
1289 }
1290 if let Some(input) = self.input.as_mut()
1291 && input.keyboard_detach().await.is_err() {
1292 self.drop_input();
1293 }
1294 self.invalidate_readings();
1297
1298 if submit_error.is_some() {
1299 self.drop_input();
1300 }
1301 let screen_changed = self
1304 .observe(keys_began, AFTER_PASTE)
1305 .await
1306 .map(|(_, after)| after.signature() != screen.signature());
1307 let mut result = observation.result(
1308 &args,
1309 &field,
1310 restored,
1311 submitted,
1312 screen_changed,
1313 submit_error,
1314 );
1315 if !restored && saved.unreadable() {
1316 let out = match &mut result {
1317 Ok(out) => out,
1318 Err(fail) => &mut fail.extra,
1319 };
1320 out.insert(
1321 "clipboard_note".into(),
1322 json!(t(
1323 "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1324 "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1325 )),
1326 );
1327 }
1328 result
1329 }
1330}
1331
1332const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1333
1334const SERVICE_KINDS: [&str; 6] = [
1345 "public.utf8-plain-text",
1346 "public.plain-text",
1347 "public.text",
1348 "public.png",
1349 "public.jpeg",
1350 "public.tiff",
1351];
1352const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1354
1355fn app_provided(types: &[String]) -> bool {
1358 types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1359}
1360
1361#[derive(Debug, PartialEq)]
1363enum Peek {
1364 Empty,
1365 Text(String),
1366 Image(String, Vec<u8>),
1367 AppContent,
1369 Unknown,
1371}
1372
1373struct Clipboard {
1375 before: Peek,
1376}
1377
1378impl Clipboard {
1379 async fn connect(
1382 adapter: &crate::tcp::handle::AdapterHandle,
1383 port: u16,
1384 ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1385 use idevice::RsdService;
1386 let mut adapter = adapter.clone();
1387 let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1388 .await
1389 .map_err(|_| "clipboard service: no answer".to_string())?
1390 .map_err(|e| format!("clipboard service: {e}"))?;
1391 idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1392 .await
1393 .map_err(|e| format!("clipboard service: {e:?}"))
1394 }
1395
1396 async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1399 use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1400 let general = idevice::core_device::GENERAL_PASTEBOARD;
1401 let Ok(mut clip) = Self::connect(adapter, port).await else {
1402 return Peek::Unknown;
1403 };
1404 let pulled = tokio::time::timeout(
1405 Duration::from_secs(8),
1406 clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1407 )
1408 .await;
1409 let Ok(Ok(snap)) = pulled else {
1410 return Peek::Unknown;
1411 };
1412 trace(|| {
1413 let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1414 format!("clipboard: kinds [{}]", kinds.join(" | "))
1415 });
1416 let Some(item) = snap.items.first() else {
1417 return Peek::Empty;
1418 };
1419 if snap.items.iter().any(|i| app_provided(&i.types)) {
1420 return Peek::AppContent;
1421 }
1422 for kind in SERVICE_KINDS {
1424 let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1425 continue;
1426 };
1427 let bytes = match &entry.payload {
1428 PasteboardPayload::Inline(b) => Some(b.clone()),
1429 PasteboardPayload::Promised { .. } => {
1430 match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1431 Ok(Ok(b)) => b,
1432 Ok(Err(_)) => None,
1433 Err(_) => return Peek::Unknown,
1435 }
1436 }
1437 PasteboardPayload::Error(_) => None,
1438 };
1439 let Some(bytes) = bytes else { continue };
1440 return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1441 Peek::Image(kind.to_string(), bytes)
1442 } else {
1443 match String::from_utf8(bytes) {
1444 Ok(text) => Peek::Text(text),
1445 Err(_) => continue,
1446 }
1447 };
1448 }
1449 Peek::Unknown
1450 }
1451
1452 async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1453 let mut clip = Self::connect(adapter, port).await?;
1454 let general = idevice::core_device::GENERAL_PASTEBOARD;
1455 match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1456 Ok(Ok(())) => Ok(()),
1457 Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1458 Err(_) => Err("set clipboard: no answer".into()),
1459 }
1460 }
1461
1462 async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1463 Clipboard { before: Self::peek(adapter, port).await }
1464 }
1465
1466 fn unreadable(&self) -> bool {
1469 matches!(self.before, Peek::AppContent | Peek::Unknown)
1470 }
1471
1472 async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1475 if typed.is_empty() {
1476 return true;
1477 }
1478 tokio::time::sleep(Duration::from_millis(150)).await;
1480 if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1483 return false;
1484 }
1485 let general = idevice::core_device::GENERAL_PASTEBOARD;
1486 let wrote = match &self.before {
1487 Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1488 Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1489 Ok(mut clip) => {
1490 matches!(
1491 tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1492 Ok(Ok(()))
1493 )
1494 }
1495 Err(_) => false,
1496 },
1497 Peek::Empty | Peek::AppContent | Peek::Unknown => {
1500 let _ = Self::set_text(adapter, port, "").await;
1501 return matches!(self.before, Peek::Empty);
1502 }
1503 };
1504 if !wrote {
1505 return false;
1506 }
1507 match (Self::peek(adapter, port).await, &self.before) {
1508 (Peek::Text(now), Peek::Text(old)) => &now == old,
1509 (Peek::Image(..), Peek::Image(..)) => true,
1510 _ => false,
1511 }
1512 }
1513}
1514
1515#[cfg(test)]
1516mod type_tests {
1517 use super::*;
1518 use crate::reply::envelope;
1519
1520 fn field(before: Option<&str>) -> Element {
1521 serde_json::from_value(json!({
1522 "id": "original", "role": "text_field", "value": before, "focused": true,
1523 }))
1524 .unwrap()
1525 }
1526
1527 fn result(
1528 args: &TypeArgs,
1529 before: Option<&str>,
1530 after: Option<&str>,
1531 screen: Option<bool>,
1532 ) -> Value {
1533 let field = field(before);
1534 let observed = TypeObservation::new(args, field.value.clone(), after.map(String::from));
1535 let submit = args.submit && observed.can_submit();
1536 envelope(
1537 observed.result(args, &field, true, Some(submit), screen, None),
1538 Instant::now(),
1539 )
1540 }
1541
1542 #[test]
1543 fn changed_values_are_observations_not_exact_text_matches() {
1544 for value in ["30000", "₩ 30,000", "•••••", "corrected text", " \n "] {
1545 let args = TypeArgs {
1546 text: "30000".into(),
1547 submit: true,
1548 ..Default::default()
1549 };
1550 let out = result(&args, Some(""), Some(value), Some(true));
1551 assert_eq!(out["ok"], true, "{out}");
1552 assert_eq!(out["outcome"], "done");
1553 assert_eq!(out["verified"], true);
1554 assert_eq!(out["value_before"], "");
1555 assert_eq!(out["value"], value);
1556 assert_eq!(out["value_changed"], true);
1557 assert_eq!(out["screen_changed"], true);
1558 assert_eq!(out["submit_sent"], true);
1559 assert!(out.get("warning").is_none());
1560 }
1561 }
1562
1563 #[test]
1564 fn unchanged_paste_warns_without_claiming_permission_was_denied() {
1565 let args = TypeArgs {
1566 text: "테스트 문장".into(),
1567 ..Default::default()
1568 };
1569 for value in ["", "old text", "•••••"] {
1570 let out = result(&args, Some(value), Some(value), Some(false));
1571 assert_eq!(out["ok"], true);
1572 assert_eq!(out["outcome"], "done");
1573 assert_eq!(out["sent"], true);
1574 assert_eq!(out["verified"], true, "verified still means read back");
1575 assert_eq!(out["value_changed"], false);
1576 assert_eq!(out["screen_changed"], false);
1577 assert_eq!(out["warning"]["code"], "value_unchanged");
1578 assert!(
1579 out["warning"]["message"]
1580 .as_str()
1581 .unwrap()
1582 .contains("may not")
1583 );
1584 assert!(out["warning"]["hint"].as_str().is_some());
1585 }
1586 let out = result(&args, Some(""), Some(""), Some(true));
1588 assert_eq!(out["value_changed"], false);
1589 assert_eq!(out["screen_changed"], true);
1590 assert!(out.get("warning").is_some());
1591 }
1592
1593 #[test]
1594 fn unreadable_and_uncomparable_values_do_not_become_empty_strings() {
1595 let args = TypeArgs {
1596 text: "hello".into(),
1597 ..Default::default()
1598 };
1599 let out = result(&args, Some("old"), None, None);
1600 assert_eq!(out["ok"], true);
1601 assert_eq!(out["verified"], false);
1602 assert!(out["value"].is_null());
1603 assert!(out["value_changed"].is_null());
1604 assert!(out["screen_changed"].is_null());
1605 assert_eq!(out["warning"]["code"], "value_unreadable");
1606 let out = result(&args, None, Some("hello"), Some(true));
1607 assert_eq!(out["verified"], true);
1608 assert!(out["value_before"].is_null());
1609 assert!(out["value_changed"].is_null());
1610 assert_eq!(out["warning"]["code"], "value_change_unknown");
1611 let out = result(&args, Some(""), Some("hello"), None);
1613 assert_eq!(out["verified"], true);
1614 assert!(out["screen_changed"].is_null());
1615 }
1616
1617 #[test]
1618 fn submit_is_withheld_when_the_input_effect_needs_review() {
1619 let args = TypeArgs {
1620 text: "hello".into(),
1621 submit: true,
1622 ..Default::default()
1623 };
1624 for (before, after, code) in [
1625 (Some(""), Some(""), "value_unchanged"),
1626 (Some("old"), Some("old"), "value_unchanged"),
1627 (Some("old"), None, "value_unreadable"),
1628 (None, Some("hello"), "value_change_unknown"),
1629 (Some("old"), Some(""), "value_empty"),
1630 ] {
1631 let out = result(&args, before, after, Some(false));
1632 assert_eq!(out["ok"], false, "{out}");
1633 assert_eq!(out["outcome"], "unknown");
1634 assert_eq!(out["sent"], true);
1635 assert_eq!(out["submit_sent"], false);
1636 assert_eq!(out["error"]["code"], "type_not_verified");
1637 assert_eq!(out["warning"]["code"], code);
1638 assert_eq!(out["value"], json!(after));
1639 assert_eq!(out["clipboard_restored"], true);
1640 }
1641 }
1642
1643 #[test]
1644 fn clear_and_identical_replacement_are_reported_separately() {
1645 let mut args = TypeArgs {
1646 clear: true,
1647 submit: true,
1648 ..Default::default()
1649 };
1650 for before in [Some(""), Some("old"), None] {
1651 let out = result(&args, before, Some(""), Some(false));
1652 assert_eq!(out["ok"], true);
1653 assert_eq!(out["submit_sent"], true);
1654 assert!(out.get("warning").is_none());
1655 }
1656 let out = result(&args, Some("old"), Some("old"), Some(false));
1657 assert_eq!(out["warning"]["code"], "clear_not_observed");
1658 assert_eq!(out["submit_sent"], false);
1659 args.text = "same".into();
1662 let out = result(&args, Some("same"), Some("same"), Some(false));
1663 assert_eq!(out["outcome"], "unknown");
1664 assert_eq!(out["warning"]["code"], "value_unchanged");
1665 }
1666
1667 #[test]
1668 fn failed_return_keeps_observations_and_does_not_claim_it_was_unsent() {
1669 let args = TypeArgs {
1670 text: "hello".into(),
1671 submit: true,
1672 ..Default::default()
1673 };
1674 let field = field(Some(""));
1675 let observed = TypeObservation::new(&args, field.value.clone(), Some("hello".into()));
1676 let out = envelope(
1677 observed.result(
1678 &args,
1679 &field,
1680 false,
1681 None,
1682 None,
1683 Some("write failed".into()),
1684 ),
1685 Instant::now(),
1686 );
1687 assert_eq!(out["outcome"], "unknown");
1688 assert_eq!(out["error"]["code"], "input_failed");
1689 assert!(out["submit_sent"].is_null());
1690 assert_eq!(out["value_changed"], true);
1691 assert_eq!(out["clipboard_restored"], false);
1692 }
1693}
1694
1695#[cfg(test)]
1696mod clipboard_tests {
1697 use super::app_provided;
1698
1699 fn kinds(k: &[&str]) -> Vec<String> {
1700 k.iter().map(|s| s.to_string()).collect()
1701 }
1702
1703 #[test]
1704 fn notes_copy_is_app_content() {
1705 assert!(app_provided(&kinds(&[
1707 "public.utf8-plain-text",
1708 "com.apple.notes.richtext",
1709 "com.apple.webarchive",
1710 "com.apple.flat-rtfd",
1711 "iOS rich content paste pasteboard type",
1712 "public.html",
1713 "net.daringfireball.markdown",
1714 "public.rtf",
1715 ])));
1716 }
1717
1718 #[test]
1719 fn what_iph_puts_there_is_not() {
1720 assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1722 assert!(!app_provided(&kinds(&["public.png"])));
1723 }
1724}