1use std::{
5 collections::BTreeMap,
6 time::{Duration, Instant},
7};
8
9use serde::Deserialize;
10use serde_json::{Value, json};
11use unicode_normalization::UnicodeNormalization;
12
13use crate::{
14 ax::{
15 attrs,
16 dump::ObjKey,
17 tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
18 },
19 device::{
20 ax::trace,
21 input::{Motion, normalize},
22 pinch,
23 session::{Fresh, Gesture, Session, reuse},
24 },
25 fields,
26 i18n::t,
27 reply::{Fail, Res},
28 selector::{Selector, dedupe_targets},
29};
30
31pub const AFTER_TAP: Duration = Duration::from_millis(480);
35pub const AFTER_DRAG: Duration = Duration::from_millis(300);
37pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
38pub const AFTER_PASTE: Duration = Duration::from_millis(200);
39pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
43pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
45const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
49const PASTE_SHOWS: Duration = Duration::from_millis(1500);
51const ASK_WAIT: Duration = Duration::from_millis(1000);
53const UNASKED_MEMORY: Duration = Duration::from_secs(5);
56const UNREAD_NOTE: &str =
58 "the screen could not be read; sent on the last known screen size, not verified";
59
60#[derive(Debug, Default, Deserialize)]
61#[serde(default, deny_unknown_fields)]
62pub struct ReadArgs {
63 pub app: bool,
65 pub all: bool,
67 pub full: bool,
69 pub raw: bool,
71 pub max_age_ms: Option<u64>,
73}
74
75#[derive(Debug, Deserialize)]
76#[serde(deny_unknown_fields)]
77pub struct FindArgs {
78 pub selector: Value,
79 #[serde(default)]
80 pub all: bool,
81}
82
83#[derive(Debug, Default, Deserialize)]
84#[serde(default, deny_unknown_fields)]
85pub struct TapArgs {
86 pub target: Value,
88 pub long_ms: Option<u64>,
89 pub double: bool,
90 pub no_wait: bool,
93}
94
95#[derive(Debug, Default, Deserialize)]
96#[serde(default, deny_unknown_fields)]
97pub struct SwipeArgs {
98 pub direction: Option<String>,
99 pub from: Option<[f64; 2]>,
100 pub to: Option<[f64; 2]>,
101 pub duration_ms: Option<u64>,
102 pub drag: bool,
104}
105
106#[derive(Debug, Default, Deserialize)]
107#[serde(default, deny_unknown_fields)]
108pub struct PinchArgs {
109 pub direction: Option<String>,
111 pub scale: Option<f64>,
113 pub rotate: Option<f64>,
115 pub at: Value,
118 pub duration_ms: Option<u64>,
119}
120
121impl PinchArgs {
122 fn motion(&self) -> Res<(f64, f64)> {
124 let scale = match (self.direction.as_deref(), self.scale) {
125 (Some(_), Some(_)) => return Err(Fail::invalid("a direction and a scale cannot be combined")),
126 (Some("in"), None) => 0.5,
127 (Some("out"), None) => 2.0,
128 (Some(other), None) => {
129 return Err(Fail::invalid(format!("unknown direction `{other}` (in, out)")));
130 }
131 (None, scale) => scale.unwrap_or(1.0),
132 };
133 let rotate = self.rotate.unwrap_or(0.0);
134 pinch::check_motion(scale, rotate).map_err(Fail::invalid)?;
135 Ok((scale, rotate))
136 }
137}
138
139#[derive(Debug, Default, Deserialize)]
140#[serde(default, deny_unknown_fields)]
141pub struct ScrollArgs {
142 pub direction: String,
143 pub until: Option<Value>,
144 pub max: Option<u32>,
145}
146
147#[derive(Debug, Default, Deserialize)]
148#[serde(default, deny_unknown_fields)]
149pub struct TypeArgs {
150 pub text: String,
151 pub clear: bool,
152 pub submit: bool,
153}
154
155struct TypeObservation {
158 before: Option<String>,
159 value: Option<String>,
160 changed: Option<bool>,
161 warning: Option<Value>,
162}
163
164impl TypeObservation {
165 fn new(args: &TypeArgs, before: Option<String>, value: Option<String>) -> Self {
166 let changed = before.as_ref().zip(value.as_ref()).map(|(a, b)| a != b);
167 let retry_hint = t(
168 "입력이 반영되었을 수 있습니다. 자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요.",
169 "Input may have taken effect. Do not automatically retype; check with `iph read` or a screenshot.",
170 );
171 let warning = if value.is_none() {
172 Some(json!({
173 "code": "value_unreadable",
174 "message": "the input was sent but the field's value could not be read afterwards",
175 "hint": retry_hint,
176 }))
177 } else if args.text.is_empty() && value.as_deref() == Some("") {
178 None
180 } else if args.text.is_empty() {
181 Some(json!({
182 "code": "clear_not_observed",
183 "message": "the clear input was sent but the field still shows a nonempty value",
184 "hint": retry_hint,
185 }))
186 } else if changed == Some(false) {
187 Some(json!({
188 "code": "value_unchanged",
189 "message": "the field's value is unchanged; the paste may not have taken effect (identical replacement, masking or app behavior can also cause this)",
190 "hint": t(
191 "자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요. 붙여넣기가 막혔을 수도 있습니다: 설정 → 앱 → 해당 앱 → 다른 앱에서 붙여넣기 → 허용. 값이 같다는 사실만으로 권한 거부를 확정할 수는 없습니다.",
192 "Do not automatically retype; check with `iph read` or a screenshot. Pasting may be blocked: Settings → Apps → this app → Paste from Other Apps → Allow. An unchanged value alone does not prove permission was denied.",
193 ),
194 }))
195 } else if value.as_deref() == Some("") {
196 Some(json!({
197 "code": "value_empty",
198 "message": "the input was sent but the field shows an empty value; the app may have cleared it",
199 "hint": retry_hint,
200 }))
201 } else if changed.is_none() {
202 Some(json!({
203 "code": "value_change_unknown",
204 "message": "the field's value was read afterwards, but its previous value was unavailable for comparison",
205 "hint": retry_hint,
206 }))
207 } else {
208 None
209 };
210 Self {
211 before,
212 value,
213 changed,
214 warning,
215 }
216 }
217
218 fn can_submit(&self) -> bool {
219 self.warning.is_none()
220 }
221
222 fn result(
223 self,
224 args: &TypeArgs,
225 field: &Element,
226 restored: bool,
227 submitted: Option<bool>,
228 screen_changed: Option<bool>,
229 submit_error: Option<String>,
230 ) -> Res {
231 let blocked_submit = args.submit && !self.can_submit();
232 let mut out = fields! {
233 "sent": true,
234 "verified": self.value.is_some(),
235 "value_before": self.before,
236 "value": self.value,
237 "value_changed": self.changed,
238 "field": brief(field),
239 "submit_sent": submitted,
240 "clipboard_restored": restored,
241 "screen_changed": screen_changed,
242 };
243 if let Some(warning) = self.warning {
244 out.insert("warning".into(), warning);
245 }
246 let fail = if let Some(e) = submit_error {
247 Some(Fail::unknown("input_failed", e))
248 } else if blocked_submit {
249 Some(Fail::unknown(
250 "type_not_verified",
251 "the input was sent, but its effect is uncertain; Return was not sent",
252 ))
253 } else {
254 None
255 };
256 if let Some(mut fail) = fail {
257 fail.hint = Some(t(
258 "입력은 이미 나갔습니다. 자동으로 재입력하거나 제출하지 말고 `iph read`나 스크린샷으로 먼저 확인하세요.",
259 "Input has already been sent. Do not automatically retype or submit; check with `iph read` or a screenshot first.",
260 ).into());
261 fail.extra = out;
262 Err(fail)
263 } else {
264 Ok(out)
265 }
266 }
267}
268
269#[derive(Debug, Deserialize)]
270#[serde(deny_unknown_fields)]
271pub struct KeyArgs {
272 pub key: String,
273}
274
275#[derive(Debug, Deserialize)]
276#[serde(deny_unknown_fields)]
277pub struct ButtonArgs {
278 pub button: String,
279}
280
281#[derive(Debug, Default, Deserialize)]
282#[serde(deny_unknown_fields)]
283pub struct LaunchArgs {
284 pub app: String,
285 #[serde(default)]
287 pub restart: bool,
288 #[serde(default)]
290 pub arguments: Vec<String>,
291 #[serde(default)]
292 pub environment: BTreeMap<String, String>,
293}
294
295impl LaunchArgs {
296 pub fn validate(&self) -> Res<()> {
297 for key in self.environment.keys() {
298 if key.is_empty() || key.contains('=') {
299 return Err(Fail::invalid(format!(
300 "environment names must be non-empty and contain no `=`: \"{key}\""
301 )));
302 }
303 }
304 Ok(())
305 }
306
307 pub fn has_launch_options(&self) -> bool {
309 !self.arguments.is_empty() || !self.environment.is_empty()
310 }
311
312 pub fn kill_existing(&self) -> bool {
313 self.restart || self.has_launch_options()
314 }
315
316 fn environment_dictionary(&self) -> Option<plist::Dictionary> {
317 (!self.environment.is_empty()).then(|| {
318 self.environment
319 .iter()
320 .map(|(k, v)| (k.clone(), plist::Value::String(v.clone())))
321 .collect()
322 })
323 }
324}
325
326#[derive(Debug, Deserialize)]
327#[serde(deny_unknown_fields)]
328pub struct ScreenshotArgs {
329 pub path: String,
330}
331
332pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
333 let v = if args.is_null() { json!({}) } else { args.clone() };
334 serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
335}
336
337pub fn brief(el: &Element) -> Value {
338 let mut v = json!({ "id": el.id, "role": el.role });
339 if let Some(l) = &el.label {
340 v["label"] = json!(l);
341 }
342 if let Some(val) = &el.value {
343 v["value"] = json!(val);
344 }
345 if let Some(i) = &el.identifier {
346 v["identifier"] = json!(i);
347 }
348 if let Some(f) = &el.frame {
349 v["frame"] = json!(f);
350 }
351 v
352}
353
354pub enum Target {
355 Selector(Selector),
356 Point(f64, f64),
357}
358
359fn daemon_silent(read_failed: &Fail) -> bool {
362 let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
363 read_failed.code == "ax_timeout" && !locked
364}
365
366pub fn parse_target(v: &Value) -> Res<Target> {
367 if let Some(obj) = v.as_object()
368 && let (Some(x), Some(y)) = (
369 obj.get("x").and_then(|n| n.as_f64()),
370 obj.get("y").and_then(|n| n.as_f64()),
371 )
372 && obj.len() == 2 {
373 return Ok(Target::Point(x, y));
374 }
375 if let Some(s) = v.as_str() {
376 let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
377 if parts.len() == 2
378 && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
379 return Ok(Target::Point(x, y));
380 }
381 }
382 Selector::from_value(v)
383 .map(Target::Selector)
384 .map_err(|e| Fail::invalid(e.to_string()))
385}
386
387pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
389 let matches = dedupe_targets(sel.find(screen));
390 match matches.len() {
391 1 => Ok(matches[0]),
392 0 => Err(Fail::not_sent(
393 "not_found",
394 format!("no element matches {}", sel.describe()),
395 )
396 .hint(t(
397 "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
398 "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
399 ))
400 .with("selector", sel.describe())
401 .with("screen_summary", screen.summary(12))),
402 n => Err(Fail::not_sent(
403 "ambiguous_selector",
404 format!("{n} elements match {}", sel.describe()),
405 )
406 .hint(t(
407 "role=, within=, index= 으로 하나만 남게 좁히세요.",
408 "Narrow it to one with role=, within= or index=.",
409 ))
410 .with("selector", sel.describe())
411 .with(
412 "candidates",
413 Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
414 )),
415 }
416}
417
418fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
420 let frame = el.frame.ok_or_else(|| {
421 Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
422 })?;
423 let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
424 let visible = frame.intersection(&bounds).ok_or_else(|| {
425 Fail::not_sent("off_screen", "the element is outside the screen")
426 .hint(t(
427 "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
428 "Bring it on screen with `iph scroll down --until …`.",
429 ))
430 .with("element", brief(el))
431 })?;
432 let covered = |x: f64, y: f64| -> Option<&Element> {
433 let hit = screen.element_at(x, y)?;
434 if hit.id == el.id {
435 return None;
436 }
437 let inside = hit.frame.is_some_and(|h| {
440 h.x >= frame.x - 1.0
441 && h.y >= frame.y - 1.0
442 && h.x + h.w <= frame.x + frame.w + 1.0
443 && h.y + h.h <= frame.y + frame.h + 1.0
444 });
445 let encloses = hit.frame.is_some_and(|h| {
446 frame.x >= h.x - 1.0
447 && frame.y >= h.y - 1.0
448 && frame.x + frame.w <= h.x + h.w + 1.0
449 && frame.y + frame.h <= h.y + h.h + 1.0
450 });
451 (!inside && !encloses).then_some(hit)
452 };
453 let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
454 let mut blocker = None;
455 for (fx, fy) in candidates {
456 let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
457 match covered(x, y) {
458 None => return Ok((x, y)),
459 Some(b) => blocker = blocker.or(Some(b)),
460 }
461 }
462 Err(Fail::not_sent("covered", "another element covers the target")
463 .hint(t(
464 "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
465 "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
466 ))
467 .with("element", brief(el))
468 .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
469}
470
471fn nfc(s: &str) -> String {
472 s.nfc().collect()
473}
474
475fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
477 let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
478 let now = after.apps.first()?;
479 (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
480}
481
482fn is_bundle_id(app: &str) -> bool {
484 app.contains('.') && !app.contains(' ')
485}
486
487pub(crate) fn bundle_id_in(apps: &[idevice::core_device::AppListEntry], app: &str) -> Res<String> {
489 if is_bundle_id(app) {
490 return Ok(app.to_string());
491 }
492 let want = nfc(app).to_lowercase();
493 let hits: Vec<&idevice::core_device::AppListEntry> = apps
494 .iter()
495 .filter(|a| nfc(&a.name).to_lowercase() == want)
496 .collect();
497 match hits.len() {
498 1 => Ok(hits[0].bundle_identifier.clone()),
499 0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
500 .hint(t(
501 "`iph apps`로 이름과 bundle id를 확인하세요.",
502 "See names and bundle ids with `iph apps`.",
503 ))),
504 _ => Err(Fail::not_sent(
505 "ambiguous_app",
506 format!("{} apps are named \"{app}\"", hits.len()),
507 )
508 .with(
509 "candidates",
510 Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
511 )),
512 }
513}
514
515impl Session {
516 fn build_opts(all: bool, raw: bool) -> BuildOptions {
517 BuildOptions { include_offscreen: all, raw }
518 }
519
520 pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
521 let fresh = match args.max_age_ms {
522 Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
523 None => Fresh::Force,
524 };
525 let (seq, mut screen) = self
526 .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
527 .await?;
528 if args.app
529 && let Some(front) = screen.apps.first().map(|a| a.pid) {
530 screen.elements.retain(|e| e.pid == front);
531 screen.apps.retain(|a| a.pid == front);
532 }
533 let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
534 Ok(fields! {
535 "screen": {"width": screen.width, "height": screen.height},
536 "apps": screen.apps,
537 "count": screen.elements.len(),
538 "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
539 "seq": seq,
540 "elements": screen.elements,
541 })
542 }
543
544 pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
545 let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
546 let (_, screen) = self
547 .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
548 .await?;
549 let matches = sel.find(&screen);
550 let mut out = fields! {
551 "selector": sel.describe(),
552 "count": matches.len(),
553 "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
554 };
555 if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
556 out.insert(
557 "note".into(),
558 json!("this reading came without identifiers, so id= could not match; try again"),
559 );
560 }
561 Ok(out)
562 }
563
564 pub(crate) async fn observe_with(
575 &mut self,
576 since: Instant,
577 delay: Duration,
578 careful: bool,
579 ) -> Option<(u64, Screen)> {
580 tokio::time::sleep(delay).await;
581 let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
582 let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
583 let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
584 if !careful && !chatty {
585 return Some(latest);
586 }
587 for _ in 0..2 {
588 trace(|| "observe: reading again until two readings agree".into());
589 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
590 break;
591 };
592 let same = next.1.signature() == latest.1.signature();
593 latest = next;
594 if same {
595 break;
596 }
597 }
598 Some(latest)
599 }
600
601 pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
602 self.observe_with(since, delay, false).await
603 }
604
605 pub(crate) async fn observe_settled(
611 &mut self,
612 since: Instant,
613 before: &[ElementSignature],
614 ) -> Option<(u64, Screen)> {
615 let mut latest = self.observe(since, AFTER_TAP).await?;
616 while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
617 trace(|| "observe: the screen has not changed yet; reading again".into());
618 match self.screen(Fresh::Force, false, BuildOptions::default()).await {
619 Ok(next) => latest = next,
620 Err(_) => return Some(latest),
621 }
622 }
623 if latest.1.signature() == before {
624 return Some(latest);
625 }
626 while since.elapsed() < SETTLE_WAIT {
627 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
628 break;
629 };
630 let same = next.1.signature() == latest.1.signature();
631 latest = next;
632 if same {
633 break;
634 }
635 trace(|| "observe: the screen is still moving; reading again".into());
636 }
637 Some(latest)
638 }
639
640 async fn locate(
643 &mut self,
644 targets: &[Target],
645 fresh: Fresh,
646 ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
647 let need_ids = targets
648 .iter()
649 .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
650 let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
651 if screen.width <= 0.0 || screen.height <= 0.0 {
652 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
653 }
654 let mut points = Vec::with_capacity(targets.len());
655 for target in targets {
656 points.push(match target {
657 Target::Selector(sel) => {
658 let el = resolve_one(sel, &screen)?;
659 let (x, y) = tap_point(el, &screen)?;
660 (x, y, Some(el.clone()))
661 }
662 Target::Point(x, y) => {
663 if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
664 return Err(Fail::invalid(format!(
665 "point {x},{y} is outside the {}x{} screen",
666 screen.width, screen.height
667 )));
668 }
669 (*x, *y, screen.element_at(*x, *y).cloned())
670 }
671 });
672 }
673 Ok((screen, points))
674 }
675
676 async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
683 let age = self.reading_age()?;
684 if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
685 return None;
686 }
687 let (_, dump, _) = self.ax.as_ref()?.latest()?;
688 let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
689 let ax = self.ax.as_mut()?;
690 for app in &dump.applications {
691 let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
692 let now: Vec<ObjKey> = ax
693 .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
694 .await?
695 .as_array()?
696 .iter()
697 .filter_map(ObjKey::from_ka)
698 .collect();
699 if now != was {
700 trace(|| "locate: the screen has changed since the last reading".into());
701 return None;
702 }
703 }
704 for (_, _, el) in &located.1 {
705 let el = el.as_ref()?;
706 let now = ax
707 .ask(el.key?, attrs::FRAME, ASK_WAIT)
708 .await?
709 .get("NS.rectval")
710 .and_then(|v| v.as_str())
711 .and_then(Rect::parse)?;
712 let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
713 if el.frame.as_ref().map(round) != Some(round(&now)) {
714 trace(|| "locate: a target has moved since the last reading".into());
715 return None;
716 }
717 }
718 trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
719 Some(located)
720 }
721
722 async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
727 let located = match self.locate_confirmed(targets).await {
728 Some(l) => l,
729 None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
730 };
731 let started = std::time::Instant::now();
732 self.ensure_input().await?;
733 if started.elapsed() < Duration::from_millis(400) {
734 return Ok(located);
735 }
736 self.locate(targets, Fresh::Force).await
737 }
738
739 pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
740 if args.double && args.long_ms.is_some() {
741 return Err(Fail::invalid("--double and --long cannot be combined"));
742 }
743 if let Some(ms) = args.long_ms
744 && !(1..=60_000).contains(&ms)
745 {
746 return Err(Fail::invalid("--long must be 1..60000 ms"));
747 }
748 let target = parse_target(&args.target)?;
749 let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
750 Ok(located) => located,
751 Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
752 };
753 let (x, y, element) = points.remove(0);
754 let element_json = element.as_ref().map(brief);
755
756 let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
757 let before = screen.signature();
758 let input_began = Instant::now();
759 let gesture = if args.double {
760 Gesture::DoubleTap { x: nx, y: ny }
761 } else {
762 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
763 };
764 self.send_gesture(gesture).await?;
765 let after = if args.no_wait {
766 self.observe(input_began, AFTER_TAP).await
767 } else {
768 self.observe_settled(input_began, &before).await
769 };
770 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
771 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
772 if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
773 self.input_suspect = true;
775 }
776
777 let mut out = fields! {
778 "sent": true,
779 "verified": changed.is_some(),
780 "screen_changed": changed.unwrap_or(false),
781 "x": (x * 10.0).round() / 10.0,
782 "y": (y * 10.0).round() / 10.0,
783 };
784 if let Some(e) = element_json {
785 out.insert("element".into(), e);
786 }
787 if changed.is_none() {
788 out.insert(
789 "note".into(),
790 json!("the tap was sent but no reading came back to compare"),
791 );
792 }
793 if let Some(change) = app_change {
794 out.insert("app_changed".into(), json!(true));
795 out.insert("app".into(), change);
796 }
797 if !args.no_wait
799 && let Some((seq, screen)) = after {
800 out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
801 out.insert("apps".into(), json!(screen.apps));
802 out.insert("count".into(), json!(screen.elements.len()));
803 out.insert("seq".into(), json!(seq));
804 out.insert("elements".into(), json!(screen.elements));
805 }
806 Ok(out)
807 }
808
809 fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
816 if !daemon_silent(read_failed) {
817 return None;
818 }
819 self.screen_size
820 }
821
822 async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
826 let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
827 return Err(read_failed);
828 };
829 let (x, y) = (*x, *y);
830 if x < 0.0 || y < 0.0 || x > w || y > h {
831 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
832 }
833 let (nx, ny) = (normalize(x, w), normalize(y, h));
834 let gesture = if args.double {
835 Gesture::DoubleTap { x: nx, y: ny }
836 } else {
837 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
838 };
839 self.send_gesture(gesture).await?;
840 Ok(fields! {
841 "sent": true,
842 "verified": false,
843 "screen_changed": false,
844 "x": (x * 10.0).round() / 10.0,
845 "y": (y * 10.0).round() / 10.0,
846 "note": UNREAD_NOTE,
847 })
848 }
849
850 pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
855 if targets.is_empty() {
856 return Err(Fail::invalid("the tap list is empty"));
857 }
858 let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
859 let (screen, points) = self.locate_ready(&parsed).await?;
860 let before = screen.signature();
861 let normalized: Vec<(u16, u16)> = points
862 .iter()
863 .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
864 .collect();
865 let tapped: Vec<Value> = points
866 .iter()
867 .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
868 .collect();
869 let input_began = Instant::now();
870 for (i, (nx, ny)) in normalized.iter().enumerate() {
871 let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
872 if let Err(mut fail) = self.send_gesture(tap).await {
873 if i > 0 {
874 fail.outcome = crate::reply::Outcome::Unknown;
876 }
877 return Err(fail
878 .hint(t(
879 "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
880 "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
881 ))
882 .with("taps_sent", i));
883 }
884 }
885 let after = self.observe_with(input_began, AFTER_TAP, true).await;
886 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
887 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
888 let mut out = fields! {
889 "sent": true,
890 "verified": changed.is_some(),
891 "screen_changed": changed.unwrap_or(false),
892 "taps": tapped.len(),
893 "elements": tapped,
894 };
895 if let Some(change) = app_change {
896 out.insert("app_changed".into(), json!(true));
897 out.insert("app".into(), change);
898 }
899 Ok(out)
900 }
901
902 pub(crate) async fn drag_points(
905 &mut self,
906 from: (f64, f64),
907 to: (f64, f64),
908 duration: Duration,
909 motion: Motion,
910 ) -> Res<(bool, Option<Screen>)> {
911 let (_, screen) = self
912 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
913 .await?;
914 if screen.width <= 0.0 || screen.height <= 0.0 {
915 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
916 }
917 for (x, y) in [from, to] {
918 if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
919 return Err(Fail::invalid(format!(
920 "point {x},{y} is outside the {}x{} screen",
921 screen.width, screen.height
922 )));
923 }
924 }
925 let before = screen.signature();
926 let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
927 let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
928 let input_began = Instant::now();
929 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
930 let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
931 if motion == Motion::Flick {
932 for _ in 0..3 {
934 let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
935 let still = match (&after, &next) {
936 (Some(a), Some(b)) => a.signature() == b.signature(),
937 _ => true,
938 };
939 after = next.or(after);
940 if still {
941 break;
942 }
943 }
944 }
945 let changed = after.as_ref().is_some_and(|s| s.signature() != before);
946 Ok((changed, after))
947 }
948
949 async fn screen_size(&mut self) -> Res<(f64, f64)> {
950 let (_, screen) = self
951 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
952 .await?;
953 Ok((screen.width, screen.height))
954 }
955
956 pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
957 let read = self
958 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
959 .await;
960 let (w, h) = match &read {
961 Ok((_, screen)) => (screen.width, screen.height),
962 Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
963 };
964 let (from, to) = match (&args.direction, args.from, args.to) {
965 (Some(dir), None, None) => {
966 let (cx, cy) = (w / 2.0, h / 2.0);
968 match dir.as_str() {
969 "up" => ((cx, h * 0.70), (cx, h * 0.30)),
970 "down" => ((cx, h * 0.30), (cx, h * 0.70)),
971 "left" => ((w * 0.80, cy), (w * 0.20, cy)),
972 "right" => ((w * 0.20, cy), (w * 0.80, cy)),
973 other => {
974 return Err(Fail::invalid(format!(
975 "unknown direction `{other}` (up, down, left, right)"
976 )));
977 }
978 }
979 }
980 (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
981 _ => return Err(Fail::invalid("give a direction, or both from and to")),
982 };
983 let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
984 let motion = if args.drag { Motion::Place } else { Motion::Flick };
985 if read.is_err() {
986 for (x, y) in [from, to] {
988 if x < 0.0 || y < 0.0 || x > w || y > h {
989 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
990 }
991 }
992 let a = (normalize(from.0, w), normalize(from.1, h));
993 let b = (normalize(to.0, w), normalize(to.1, h));
994 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
995 return Ok(fields! {
996 "sent": true,
997 "verified": false,
998 "screen_changed": false,
999 "from": [from.0, from.1],
1000 "to": [to.0, to.1],
1001 "note": UNREAD_NOTE,
1002 });
1003 }
1004 let (changed, after) = self.drag_points(from, to, duration, motion).await?;
1005 Ok(fields! {
1006 "sent": true,
1007 "verified": after.is_some(),
1008 "screen_changed": changed,
1009 "from": [from.0, from.1],
1010 "to": [to.0, to.1],
1011 })
1012 }
1013
1014 pub async fn cmd_pinch(&mut self, args: PinchArgs) -> Res {
1019 let (scale, rotate) = args.motion()?;
1020 let duration = Duration::from_millis(args.duration_ms.unwrap_or(500).clamp(100, 10_000));
1021 let target = (!args.at.is_null()).then(|| parse_target(&args.at)).transpose()?;
1022 let read = match &target {
1023 Some(target) => self.locate_ready(std::slice::from_ref(target)).await.map(|(screen, mut points)| {
1024 let (x, y, _) = points.remove(0);
1025 (screen, Some((x, y)))
1026 }),
1027 None => self
1028 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1029 .await
1030 .map(|(_, screen)| (screen, None)),
1031 };
1032 let (size, at, before) = match &read {
1033 Ok((screen, _)) if screen.width <= 0.0 || screen.height <= 0.0 => {
1034 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
1035 }
1036 Ok((screen, at)) => ((screen.width, screen.height), *at, Some(screen.signature())),
1037 Err(read_failed) => {
1038 let size = self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?;
1040 let at = match &target {
1041 None => None,
1042 Some(Target::Point(x, y)) => Some((*x, *y)),
1043 Some(Target::Selector(_)) => return Err(read_failed.clone()),
1044 };
1045 (size, at, None)
1046 }
1047 };
1048 let center = at.unwrap_or((size.0 / 2.0, size.1 / 2.0));
1049 let plan = pinch::plan(&pinch::Spec { screen: size, center, scale, rotate, duration }).map_err(Fail::invalid)?;
1050 let span = [(plan.span.0 * 10.0).round() / 10.0, (plan.span.1 * 10.0).round() / 10.0];
1051 let input_began = Instant::now();
1052 self.send_gesture(Gesture::Pinch(plan)).await?;
1053 let mut out = fields! {
1054 "sent": true,
1055 "center": [(center.0 * 10.0).round() / 10.0, (center.1 * 10.0).round() / 10.0],
1056 "scale": scale,
1057 "rotate": rotate,
1058 "span": span,
1059 };
1060 let Some(before) = before else {
1061 out.insert("verified".into(), json!(false));
1062 out.insert("screen_changed".into(), json!(false));
1063 out.insert("note".into(), json!(UNREAD_NOTE));
1064 return Ok(out);
1065 };
1066 let after = self.observe(input_began, AFTER_DRAG).await.map(|(_, screen)| screen);
1067 out.insert("verified".into(), json!(after.is_some()));
1068 out.insert("screen_changed".into(), json!(after.is_some_and(|s| s.signature() != before)));
1069 Ok(out)
1070 }
1071
1072 async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
1074 let (w, h) = self.screen_size().await?;
1075 let cx = w / 2.0;
1076 let (from, to) = match direction {
1077 "down" => ((cx, h * 0.68), (cx, h * 0.32)),
1078 "up" => ((cx, h * 0.32), (cx, h * 0.68)),
1079 other => {
1080 return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
1081 }
1082 };
1083 self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
1086 .await
1087 }
1088
1089 pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
1090 if args.direction != "up" && args.direction != "down" {
1091 return Err(Fail::invalid("direction must be up or down"));
1092 }
1093 let Some(until) = &args.until else {
1094 let (changed, after) = self.scroll_step(&args.direction).await?;
1095 return Ok(fields! {
1096 "sent": true,
1097 "verified": after.is_some(),
1098 "screen_changed": changed,
1099 "scrolls": 1,
1100 });
1101 };
1102 let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
1103 let max = args.max.unwrap_or(10).clamp(1, 100);
1104 let full = sel.needs_identifier();
1105 let mut scrolls = 0u32;
1106 let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
1107 loop {
1108 let matches = dedupe_targets(sel.find(&screen));
1109 if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
1111 if matches.len() > 1 {
1112 return Err(Fail::not_sent(
1113 "ambiguous_selector",
1114 format!("{} elements match {}", matches.len(), sel.describe()),
1115 )
1116 .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
1117 .with("scrolls", scrolls));
1118 }
1119 return Ok(fields! {
1120 "found": true,
1121 "scrolls": scrolls,
1122 "screen_changed": scrolls > 0,
1123 "element": brief(el),
1124 });
1125 }
1126 if scrolls >= max {
1127 return Err(Fail::new(
1128 crate::reply::Outcome::Done,
1129 "not_found",
1130 format!("{} not found after {scrolls} scrolls", sel.describe()),
1131 )
1132 .with("found", false)
1133 .with("scrolls", scrolls)
1134 .with("screen_summary", screen.summary(12)));
1135 }
1136 let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
1137 scrolls += 1;
1138 if !changed {
1139 (changed, after) = self.scroll_step(&args.direction).await?;
1142 scrolls += 1;
1143 }
1144 if !changed {
1145 return Err(Fail::new(
1146 crate::reply::Outcome::Done,
1147 "not_found",
1148 format!(
1149 "{} not found; the list stopped moving after {scrolls} scrolls",
1150 sel.describe()
1151 ),
1152 )
1153 .with("found", false)
1154 .with("scrolls", scrolls)
1155 .with("screen_summary", screen.summary(12)));
1156 }
1157 screen = match (after, full) {
1158 (Some(s), false) => s,
1159 _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
1160 };
1161 }
1162 }
1163
1164 pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
1165 let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
1166 let (_, screen) = self
1167 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1168 .await?;
1169 let before = Some(screen.signature());
1170 let input_began = Instant::now();
1171 let input = self.ensure_input().await?;
1172 if let Err(e) = input.key(usage, &modifiers).await {
1173 self.drop_input();
1174 return Err(Fail::unknown("input_failed", format!("{e:#}")));
1175 }
1176 let changed = match before {
1177 Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
1178 None => None,
1179 };
1180 Ok(fields! {
1181 "sent": true,
1182 "verified": changed.is_some(),
1183 "screen_changed": changed.unwrap_or(false),
1184 "key": args.key,
1185 })
1186 }
1187
1188 pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
1189 let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
1190 let before = match self
1192 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1193 .await
1194 {
1195 Ok((_, screen)) => Some(screen.signature()),
1196 Err(read_failed) if daemon_silent(&read_failed) => None,
1197 Err(read_failed) => return Err(read_failed),
1198 };
1199 let input_began = Instant::now();
1200 self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
1201 let changed = match &before {
1202 Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
1203 None => None,
1204 };
1205 let mut out = fields! {
1206 "sent": true,
1207 "verified": changed.is_some(),
1208 "screen_changed": changed.unwrap_or(false),
1209 "button": args.button,
1210 };
1211 if before.is_none() {
1212 out.insert("note".into(), json!(UNREAD_NOTE));
1213 }
1214 Ok(out)
1215 }
1216
1217 pub async fn cmd_apps(&mut self) -> Res {
1218 let apps = self.list_apps(true).await?;
1219 let list: Vec<Value> = apps
1220 .iter()
1221 .map(|a| {
1222 json!({
1223 "bundle_id": a.bundle_identifier,
1224 "name": a.name,
1225 "version": a.version,
1226 "first_party": a.is_first_party,
1227 "removable": a.is_removable,
1228 })
1229 })
1230 .collect();
1231 Ok(fields! { "count": list.len(), "apps": list })
1232 }
1233
1234 pub(crate) async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
1235 let apps = if is_bundle_id(app) { Vec::new() } else { self.list_apps(false).await? };
1236 bundle_id_in(&apps, app)
1237 }
1238
1239 pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
1240 args.validate()?;
1241 let bundle = self.resolve_bundle(&args.app).await?;
1242 let mut svc = self.app_service().await?;
1243 let arguments: Vec<&str> = args.arguments.iter().map(String::as_str).collect();
1244 let launch_began = Instant::now();
1245 let launched = tokio::time::timeout(
1247 Duration::from_secs(30),
1248 svc.launch_application(
1249 bundle.as_str(),
1250 &arguments,
1251 args.kill_existing(),
1252 false,
1253 args.environment_dictionary(),
1254 None,
1255 None,
1256 ),
1257 )
1258 .await
1259 .map_err(|_| "no answer within 30 s".to_string())
1260 .and_then(|r| r.map_err(|e| format!("{e:?}")));
1261 let resp = match launched {
1262 Ok(r) => r,
1263 Err(msg) => {
1264 let locked = msg.to_lowercase().contains("lock");
1265 return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1266 t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1267 } else {
1268 t(
1269 "bundle id가 맞는지 `iph apps`로 확인하세요.",
1270 "Check the bundle id with `iph apps`.",
1271 )
1272 }));
1273 }
1274 };
1275 let pid = resp.pid as i64;
1276 let mut foreground = false;
1278 let mut delay = AFTER_LAUNCH;
1279 for _ in 0..4 {
1280 if let Some((_, screen)) = self.observe(launch_began, delay).await
1281 && screen.apps.iter().any(|a| a.pid == pid) {
1282 foreground = true;
1283 break;
1284 }
1285 delay = Duration::from_millis(300);
1286 }
1287 let mut out = fields! {
1288 "sent": true,
1289 "verified": foreground,
1290 "foreground": foreground,
1291 "bundle_id": bundle,
1292 "pid": pid,
1293 "restart": args.kill_existing(),
1294 };
1295 if args.has_launch_options() {
1296 out.insert("restart_implied".into(), json!(!args.restart));
1297 out.insert("arguments".into(), json!(args.arguments));
1298 out.insert("environment_keys".into(), json!(args.environment.keys().collect::<Vec<_>>()));
1299 }
1300 Ok(out)
1301 }
1302
1303 pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1304 let png = self.screenshot_png().await?;
1305 let path = std::path::PathBuf::from(&args.path);
1306 if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1307 std::fs::create_dir_all(dir)
1308 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1309 }
1310 std::fs::write(&path, &png)
1311 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1312 Ok(fields! { "path": args.path, "bytes": png.len() })
1313 }
1314
1315 fn focused_field(screen: &Screen) -> Option<&Element> {
1317 let mut focused = screen.elements.iter().filter(|e| e.focused);
1318 let first = focused.next()?;
1319 focused.next().is_none().then_some(first)
1320 }
1321
1322 async fn field_value(&mut self, field: &Element) -> Option<String> {
1327 let key = field.key?;
1328 let before = field.value.as_deref();
1329 let began = Instant::now();
1330 loop {
1331 let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1332 let now = crate::ax::tree::value_text(Some(&value))?;
1333 if Some(now.as_str()) != before || began.elapsed() >= PASTE_SHOWS {
1334 return Some(now);
1335 }
1336 tokio::time::sleep(Duration::from_millis(100)).await;
1337 }
1338 }
1339
1340 pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1341 if args.text.is_empty() && !args.clear {
1342 return Err(Fail::invalid("nothing to type"));
1343 }
1344 if args.text.len() > 1024 * 1024 {
1345 return Err(Fail::invalid("text is larger than 1 MiB"));
1346 }
1347 let (_, screen) = self
1348 .screen(Fresh::Force, false, BuildOptions::default())
1349 .await?;
1350 let Some(field) = Self::focused_field(&screen).cloned() else {
1351 return Err(Fail::not_sent(
1352 "no_focused_field",
1353 "no text field has the keyboard right now",
1354 )
1355 .hint(t(
1356 "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1357 "Tap the field first with `iph tap` so the keyboard comes up.",
1358 ))
1359 .with("screen_summary", screen.summary(12)));
1360 };
1361 trace(|| "type: focused field found".into());
1362
1363 self.ensure_input().await?;
1366 let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1367
1368 let input = self.ensure_input().await?;
1371 let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1372 trace(|| "type: clipboard saved, keyboard attached".into());
1373 if let Err(e) = attached {
1374 return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1376 }
1377 if !args.text.is_empty()
1378 && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1379 let _ = input.keyboard_detach().await;
1380 self.invalidate_readings();
1381 return Err(Fail::not_sent("clipboard_failed", e));
1382 }
1383 trace(|| "type: clipboard set".into());
1384
1385 let cmd = vec![crate::keys::MOD_COMMAND];
1386 let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1387 if args.clear {
1388 combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1389 combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1390 }
1391 if !args.text.is_empty() {
1392 combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1393 }
1394 let keys_began = Instant::now();
1395 let pressed = input.keyboard_press(&combos).await;
1396 trace(|| "type: keys sent".into());
1397 if let Err(e) = pressed {
1398 let _ = input.keyboard_detach().await;
1399 self.drop_input();
1400 self.invalidate_readings();
1401 let restored = saved.restore(&adapter, port, &args.text).await;
1402 return Err(Fail::unknown("input_failed", format!("{e:#}"))
1403 .hint(t(
1404 "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1405 "The input may have gone out. Do not resend; check the screen with `iph read`.",
1406 ))
1407 .with("clipboard_restored", restored));
1408 }
1409
1410 let restoring = saved.restore(&adapter, port, &args.text);
1417 let reading = async {
1418 tokio::time::sleep(AFTER_PASTE).await;
1419 if let Some(value) = self.field_value(&field).await {
1420 return Some(value);
1421 }
1422 let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1423 screen.by_id(&field.id).and_then(|e| e.value.clone())
1426 };
1427 let (restored, value) = tokio::join!(restoring, reading);
1428 trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1429 let observation = TypeObservation::new(&args, field.value.clone(), value);
1430
1431 let mut submitted = Some(false);
1432 let mut submit_error = None;
1433 if args.submit && observation.can_submit() {
1434 if let Some(input) = self.input.as_mut() {
1435 match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1436 Ok(()) => submitted = Some(true),
1437 Err(e) => {
1438 submitted = None;
1439 submit_error = Some(format!("{e:#}"));
1440 }
1441 }
1442 } else {
1443 submit_error = Some("the input session closed before Return could be sent".into());
1444 }
1445 }
1446 if let Some(input) = self.input.as_mut()
1447 && input.keyboard_detach().await.is_err() {
1448 self.drop_input();
1449 }
1450 self.invalidate_readings();
1453
1454 if submit_error.is_some() {
1455 self.drop_input();
1456 }
1457 let screen_changed = self
1460 .observe(keys_began, AFTER_PASTE)
1461 .await
1462 .map(|(_, after)| after.signature() != screen.signature());
1463 let mut result = observation.result(
1464 &args,
1465 &field,
1466 restored,
1467 submitted,
1468 screen_changed,
1469 submit_error,
1470 );
1471 if !restored && saved.unreadable() {
1472 let out = match &mut result {
1473 Ok(out) => out,
1474 Err(fail) => &mut fail.extra,
1475 };
1476 out.insert(
1477 "clipboard_note".into(),
1478 json!(t(
1479 "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1480 "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1481 )),
1482 );
1483 }
1484 result
1485 }
1486}
1487
1488const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1489
1490const SERVICE_KINDS: [&str; 6] = [
1501 "public.utf8-plain-text",
1502 "public.plain-text",
1503 "public.text",
1504 "public.png",
1505 "public.jpeg",
1506 "public.tiff",
1507];
1508const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1510
1511fn app_provided(types: &[String]) -> bool {
1514 types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1515}
1516
1517#[derive(Debug, PartialEq)]
1519enum Peek {
1520 Empty,
1521 Text(String),
1522 Image(String, Vec<u8>),
1523 AppContent,
1525 Unknown,
1527}
1528
1529struct Clipboard {
1531 before: Peek,
1532}
1533
1534impl Clipboard {
1535 async fn connect(
1538 adapter: &crate::tcp::handle::AdapterHandle,
1539 port: u16,
1540 ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1541 use idevice::RsdService;
1542 let mut adapter = adapter.clone();
1543 let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1544 .await
1545 .map_err(|_| "clipboard service: no answer".to_string())?
1546 .map_err(|e| format!("clipboard service: {e}"))?;
1547 idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1548 .await
1549 .map_err(|e| format!("clipboard service: {e:?}"))
1550 }
1551
1552 async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1555 use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1556 let general = idevice::core_device::GENERAL_PASTEBOARD;
1557 let Ok(mut clip) = Self::connect(adapter, port).await else {
1558 return Peek::Unknown;
1559 };
1560 let pulled = tokio::time::timeout(
1561 Duration::from_secs(8),
1562 clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1563 )
1564 .await;
1565 let Ok(Ok(snap)) = pulled else {
1566 return Peek::Unknown;
1567 };
1568 trace(|| {
1569 let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1570 format!("clipboard: kinds [{}]", kinds.join(" | "))
1571 });
1572 let Some(item) = snap.items.first() else {
1573 return Peek::Empty;
1574 };
1575 if snap.items.iter().any(|i| app_provided(&i.types)) {
1576 return Peek::AppContent;
1577 }
1578 for kind in SERVICE_KINDS {
1580 let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1581 continue;
1582 };
1583 let bytes = match &entry.payload {
1584 PasteboardPayload::Inline(b) => Some(b.clone()),
1585 PasteboardPayload::Promised { .. } => {
1586 match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1587 Ok(Ok(b)) => b,
1588 Ok(Err(_)) => None,
1589 Err(_) => return Peek::Unknown,
1591 }
1592 }
1593 PasteboardPayload::Error(_) => None,
1594 };
1595 let Some(bytes) = bytes else { continue };
1596 return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1597 Peek::Image(kind.to_string(), bytes)
1598 } else {
1599 match String::from_utf8(bytes) {
1600 Ok(text) => Peek::Text(text),
1601 Err(_) => continue,
1602 }
1603 };
1604 }
1605 Peek::Unknown
1606 }
1607
1608 async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1609 let mut clip = Self::connect(adapter, port).await?;
1610 let general = idevice::core_device::GENERAL_PASTEBOARD;
1611 match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1612 Ok(Ok(())) => Ok(()),
1613 Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1614 Err(_) => Err("set clipboard: no answer".into()),
1615 }
1616 }
1617
1618 async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1619 Clipboard { before: Self::peek(adapter, port).await }
1620 }
1621
1622 fn unreadable(&self) -> bool {
1625 matches!(self.before, Peek::AppContent | Peek::Unknown)
1626 }
1627
1628 async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1631 if typed.is_empty() {
1632 return true;
1633 }
1634 tokio::time::sleep(Duration::from_millis(150)).await;
1636 if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1639 return false;
1640 }
1641 let general = idevice::core_device::GENERAL_PASTEBOARD;
1642 let wrote = match &self.before {
1643 Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1644 Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1645 Ok(mut clip) => {
1646 matches!(
1647 tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1648 Ok(Ok(()))
1649 )
1650 }
1651 Err(_) => false,
1652 },
1653 Peek::Empty | Peek::AppContent | Peek::Unknown => {
1656 let _ = Self::set_text(adapter, port, "").await;
1657 return matches!(self.before, Peek::Empty);
1658 }
1659 };
1660 if !wrote {
1661 return false;
1662 }
1663 match (Self::peek(adapter, port).await, &self.before) {
1664 (Peek::Text(now), Peek::Text(old)) => &now == old,
1665 (Peek::Image(..), Peek::Image(..)) => true,
1666 _ => false,
1667 }
1668 }
1669}
1670
1671#[cfg(test)]
1672mod type_tests {
1673 use super::*;
1674 use crate::reply::envelope;
1675
1676 fn field(before: Option<&str>) -> Element {
1677 serde_json::from_value(json!({
1678 "id": "original", "role": "text_field", "value": before, "focused": true,
1679 }))
1680 .unwrap()
1681 }
1682
1683 fn result(
1684 args: &TypeArgs,
1685 before: Option<&str>,
1686 after: Option<&str>,
1687 screen: Option<bool>,
1688 ) -> Value {
1689 let field = field(before);
1690 let observed = TypeObservation::new(args, field.value.clone(), after.map(String::from));
1691 let submit = args.submit && observed.can_submit();
1692 envelope(
1693 observed.result(args, &field, true, Some(submit), screen, None),
1694 Instant::now(),
1695 )
1696 }
1697
1698 #[test]
1699 fn changed_values_are_observations_not_exact_text_matches() {
1700 for value in ["30000", "₩ 30,000", "•••••", "corrected text", " \n "] {
1701 let args = TypeArgs {
1702 text: "30000".into(),
1703 submit: true,
1704 ..Default::default()
1705 };
1706 let out = result(&args, Some(""), Some(value), Some(true));
1707 assert_eq!(out["ok"], true, "{out}");
1708 assert_eq!(out["outcome"], "done");
1709 assert_eq!(out["verified"], true);
1710 assert_eq!(out["value_before"], "");
1711 assert_eq!(out["value"], value);
1712 assert_eq!(out["value_changed"], true);
1713 assert_eq!(out["screen_changed"], true);
1714 assert_eq!(out["submit_sent"], true);
1715 assert!(out.get("warning").is_none());
1716 }
1717 }
1718
1719 #[test]
1720 fn unchanged_paste_warns_without_claiming_permission_was_denied() {
1721 let args = TypeArgs {
1722 text: "테스트 문장".into(),
1723 ..Default::default()
1724 };
1725 for value in ["", "old text", "•••••"] {
1726 let out = result(&args, Some(value), Some(value), Some(false));
1727 assert_eq!(out["ok"], true);
1728 assert_eq!(out["outcome"], "done");
1729 assert_eq!(out["sent"], true);
1730 assert_eq!(out["verified"], true, "verified still means read back");
1731 assert_eq!(out["value_changed"], false);
1732 assert_eq!(out["screen_changed"], false);
1733 assert_eq!(out["warning"]["code"], "value_unchanged");
1734 assert!(
1735 out["warning"]["message"]
1736 .as_str()
1737 .unwrap()
1738 .contains("may not")
1739 );
1740 assert!(out["warning"]["hint"].as_str().is_some());
1741 }
1742 let out = result(&args, Some(""), Some(""), Some(true));
1744 assert_eq!(out["value_changed"], false);
1745 assert_eq!(out["screen_changed"], true);
1746 assert!(out.get("warning").is_some());
1747 }
1748
1749 #[test]
1750 fn unreadable_and_uncomparable_values_do_not_become_empty_strings() {
1751 let args = TypeArgs {
1752 text: "hello".into(),
1753 ..Default::default()
1754 };
1755 let out = result(&args, Some("old"), None, None);
1756 assert_eq!(out["ok"], true);
1757 assert_eq!(out["verified"], false);
1758 assert!(out["value"].is_null());
1759 assert!(out["value_changed"].is_null());
1760 assert!(out["screen_changed"].is_null());
1761 assert_eq!(out["warning"]["code"], "value_unreadable");
1762 let out = result(&args, None, Some("hello"), Some(true));
1763 assert_eq!(out["verified"], true);
1764 assert!(out["value_before"].is_null());
1765 assert!(out["value_changed"].is_null());
1766 assert_eq!(out["warning"]["code"], "value_change_unknown");
1767 let out = result(&args, Some(""), Some("hello"), None);
1769 assert_eq!(out["verified"], true);
1770 assert!(out["screen_changed"].is_null());
1771 }
1772
1773 #[test]
1774 fn submit_is_withheld_when_the_input_effect_needs_review() {
1775 let args = TypeArgs {
1776 text: "hello".into(),
1777 submit: true,
1778 ..Default::default()
1779 };
1780 for (before, after, code) in [
1781 (Some(""), Some(""), "value_unchanged"),
1782 (Some("old"), Some("old"), "value_unchanged"),
1783 (Some("old"), None, "value_unreadable"),
1784 (None, Some("hello"), "value_change_unknown"),
1785 (Some("old"), Some(""), "value_empty"),
1786 ] {
1787 let out = result(&args, before, after, Some(false));
1788 assert_eq!(out["ok"], false, "{out}");
1789 assert_eq!(out["outcome"], "unknown");
1790 assert_eq!(out["sent"], true);
1791 assert_eq!(out["submit_sent"], false);
1792 assert_eq!(out["error"]["code"], "type_not_verified");
1793 assert_eq!(out["warning"]["code"], code);
1794 assert_eq!(out["value"], json!(after));
1795 assert_eq!(out["clipboard_restored"], true);
1796 }
1797 }
1798
1799 #[test]
1800 fn clear_and_identical_replacement_are_reported_separately() {
1801 let mut args = TypeArgs {
1802 clear: true,
1803 submit: true,
1804 ..Default::default()
1805 };
1806 for before in [Some(""), Some("old"), None] {
1807 let out = result(&args, before, Some(""), Some(false));
1808 assert_eq!(out["ok"], true);
1809 assert_eq!(out["submit_sent"], true);
1810 assert!(out.get("warning").is_none());
1811 }
1812 let out = result(&args, Some("old"), Some("old"), Some(false));
1813 assert_eq!(out["warning"]["code"], "clear_not_observed");
1814 assert_eq!(out["submit_sent"], false);
1815 args.text = "same".into();
1818 let out = result(&args, Some("same"), Some("same"), Some(false));
1819 assert_eq!(out["outcome"], "unknown");
1820 assert_eq!(out["warning"]["code"], "value_unchanged");
1821 }
1822
1823 #[test]
1824 fn failed_return_keeps_observations_and_does_not_claim_it_was_unsent() {
1825 let args = TypeArgs {
1826 text: "hello".into(),
1827 submit: true,
1828 ..Default::default()
1829 };
1830 let field = field(Some(""));
1831 let observed = TypeObservation::new(&args, field.value.clone(), Some("hello".into()));
1832 let out = envelope(
1833 observed.result(
1834 &args,
1835 &field,
1836 false,
1837 None,
1838 None,
1839 Some("write failed".into()),
1840 ),
1841 Instant::now(),
1842 );
1843 assert_eq!(out["outcome"], "unknown");
1844 assert_eq!(out["error"]["code"], "input_failed");
1845 assert!(out["submit_sent"].is_null());
1846 assert_eq!(out["value_changed"], true);
1847 assert_eq!(out["clipboard_restored"], false);
1848 }
1849}
1850
1851#[cfg(test)]
1852mod clipboard_tests {
1853 use super::app_provided;
1854
1855 fn kinds(k: &[&str]) -> Vec<String> {
1856 k.iter().map(|s| s.to_string()).collect()
1857 }
1858
1859 #[test]
1860 fn notes_copy_is_app_content() {
1861 assert!(app_provided(&kinds(&[
1863 "public.utf8-plain-text",
1864 "com.apple.notes.richtext",
1865 "com.apple.webarchive",
1866 "com.apple.flat-rtfd",
1867 "iOS rich content paste pasteboard type",
1868 "public.html",
1869 "net.daringfireball.markdown",
1870 "public.rtf",
1871 ])));
1872 }
1873
1874 #[test]
1875 fn what_iph_puts_there_is_not() {
1876 assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1878 assert!(!app_provided(&kinds(&["public.png"])));
1879 }
1880}
1881
1882#[cfg(test)]
1883mod launch_tests {
1884 use super::LaunchArgs;
1885 use crate::actions::parse_args;
1886 use serde_json::json;
1887
1888 fn parse(v: serde_json::Value) -> Result<LaunchArgs, crate::reply::Fail> {
1889 parse_args::<LaunchArgs>(&v)
1890 }
1891
1892 #[test]
1893 fn a_plain_launch_sends_nothing_extra() {
1894 let a = parse(json!({"app": "x"})).unwrap();
1895 assert!(!a.kill_existing());
1896 assert!(a.environment_dictionary().is_none());
1897 }
1898
1899 #[test]
1900 fn arguments_or_environment_imply_a_restart() {
1901 let a = parse(json!({"app": "x", "arguments": ["-k", "v"]})).unwrap();
1902 assert!(a.kill_existing() && !a.restart);
1903 let b = parse(json!({"app": "x", "environment": {"TZ": "UTC"}})).unwrap();
1904 assert!(b.kill_existing());
1905 let dict = b.environment_dictionary().unwrap();
1906 assert_eq!(dict.get("TZ").and_then(|v| v.as_string()), Some("UTC"));
1907 }
1908
1909 #[test]
1910 fn environment_values_may_hold_an_equals_sign() {
1911 let a = parse(json!({"app": "x", "environment": {"K": "a=b"}})).unwrap();
1912 assert!(a.validate().is_ok());
1913 }
1914
1915 #[test]
1916 fn environment_names_are_checked() {
1917 for name in ["", "A=B"] {
1918 let a = parse(json!({"app": "x", "environment": {name: "v"}})).unwrap();
1919 assert_eq!(a.validate().unwrap_err().code, "invalid_argument");
1920 }
1921 }
1922
1923 #[test]
1924 fn a_non_string_value_is_rejected_at_the_boundary() {
1925 assert_eq!(parse(json!({"app": "x", "environment": {"A": 1}})).unwrap_err().code, "invalid_argument");
1926 assert_eq!(parse(json!({"app": "x", "environment": ["A=1"]})).unwrap_err().code, "invalid_argument");
1927 assert_eq!(parse(json!({"app": "x", "arguments": "-a"})).unwrap_err().code, "invalid_argument");
1928 assert_eq!(parse(json!({"app": "x", "arguments": [1]})).unwrap_err().code, "invalid_argument");
1929 }
1930}
1931
1932#[cfg(test)]
1933mod pinch_tests {
1934 use super::*;
1935 use crate::{config::Config, reply::Outcome};
1936
1937 fn args(v: Value) -> PinchArgs {
1938 serde_json::from_value(v).unwrap()
1939 }
1940
1941 #[test]
1942 fn a_direction_means_a_scale() {
1943 assert_eq!(args(json!({"direction": "in"})).motion().unwrap(), (0.5, 0.0));
1944 assert_eq!(args(json!({"direction": "out"})).motion().unwrap(), (2.0, 0.0));
1945 assert_eq!(args(json!({"direction": "out", "rotate": -30})).motion().unwrap(), (2.0, -30.0));
1946 assert_eq!(args(json!({"scale": 1.5, "rotate": 90})).motion().unwrap(), (1.5, 90.0));
1947 assert_eq!(args(json!({"rotate": 45})).motion().unwrap(), (1.0, 45.0));
1948 }
1949
1950 #[test]
1951 fn arguments_that_cannot_make_a_gesture_are_invalid() {
1952 for bad in [
1953 json!({}),
1954 json!({"scale": 1}),
1955 json!({"direction": "out", "scale": 2}),
1956 json!({"direction": "sideways"}),
1957 json!({"scale": 0.2}),
1958 json!({"scale": 5}),
1959 json!({"rotate": 181}),
1960 json!({"direction": "in", "rotate": -181}),
1961 ] {
1962 let fail = args(bad.clone()).motion().unwrap_err();
1963 assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1964 }
1965 assert!(serde_json::from_value::<PinchArgs>(json!({"scale": 2, "zoom": 1})).is_err());
1966 }
1967
1968 #[tokio::test]
1969 async fn a_bad_request_fails_before_the_phone_is_asked_for_anything() {
1970 let mut s = Session::new("00000000-TESTONLY-NOPHONE".into(), Config::default());
1971 for bad in [json!({}), json!({"direction": "in", "scale": 2}), json!({"scale": 9})] {
1972 let fail = s.cmd_pinch(args(bad.clone())).await.unwrap_err();
1973 assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1974 }
1975 let fail = s.cmd_pinch(args(json!({"direction": "in", "at": 7}))).await.unwrap_err();
1976 assert_eq!(fail.outcome, Outcome::NotSent);
1977 }
1978}