1use std::{
5 collections::BTreeMap,
6 time::{Duration, Instant},
7};
8
9use serde::Deserialize;
10use serde_json::{Value, json};
11use unicode_normalization::UnicodeNormalization;
12
13use crate::{
14 ax::{
15 attrs,
16 dump::ObjKey,
17 tree::{BuildOptions, Element, ElementSignature, Rect, Screen},
18 },
19 device::{
20 ax::trace,
21 input::{Motion, normalize},
22 pinch,
23 session::{Fresh, Gesture, Session, reuse},
24 },
25 fields,
26 i18n::t,
27 reply::{Fail, Res},
28 selector::{Selector, dedupe_targets},
29};
30
31pub const AFTER_TAP: Duration = Duration::from_millis(480);
35pub const AFTER_DRAG: Duration = Duration::from_millis(300);
37pub const AFTER_LAUNCH: Duration = Duration::from_millis(600);
38pub const AFTER_PASTE: Duration = Duration::from_millis(200);
39pub const CHANGE_WAIT: Duration = Duration::from_millis(2500);
43pub const SETTLE_WAIT: Duration = Duration::from_secs(5);
45const CONFIRMED_REUSE: Duration = Duration::from_secs(10);
49const PASTE_SHOWS: Duration = Duration::from_millis(1500);
51const ASK_WAIT: Duration = Duration::from_millis(1000);
53const UNASKED_MEMORY: Duration = Duration::from_secs(5);
56const UNREAD_NOTE: &str =
58 "the screen could not be read; sent on the last known screen size, not verified";
59
60#[derive(Debug, Default, Deserialize)]
61#[serde(default, deny_unknown_fields)]
62pub struct ReadArgs {
63 pub app: bool,
65 pub all: bool,
67 pub full: bool,
69 pub raw: bool,
71 pub max_age_ms: Option<u64>,
73}
74
75#[derive(Debug, Deserialize)]
76#[serde(deny_unknown_fields)]
77pub struct FindArgs {
78 pub selector: Value,
79 #[serde(default)]
80 pub all: bool,
81}
82
83#[derive(Debug, Default, Deserialize)]
84#[serde(default, deny_unknown_fields)]
85pub struct TapArgs {
86 pub target: Value,
88 pub long_ms: Option<u64>,
89 pub double: bool,
90 pub no_wait: bool,
93}
94
95#[derive(Debug, Default, Deserialize)]
96#[serde(default, deny_unknown_fields)]
97pub struct SwipeArgs {
98 pub direction: Option<String>,
99 pub from: Option<[f64; 2]>,
100 pub to: Option<[f64; 2]>,
101 pub duration_ms: Option<u64>,
102 pub drag: bool,
104}
105
106#[derive(Debug, Default, Deserialize)]
107#[serde(default, deny_unknown_fields)]
108pub struct PinchArgs {
109 pub direction: Option<String>,
111 pub scale: Option<f64>,
113 pub rotate: Option<f64>,
115 pub at: Value,
118 pub duration_ms: Option<u64>,
119}
120
121impl PinchArgs {
122 fn motion(&self) -> Res<(f64, f64)> {
124 let scale = match (self.direction.as_deref(), self.scale) {
125 (Some(_), Some(_)) => return Err(Fail::invalid("a direction and a scale cannot be combined")),
126 (Some("in"), None) => 0.5,
127 (Some("out"), None) => 2.0,
128 (Some(other), None) => {
129 return Err(Fail::invalid(format!("unknown direction `{other}` (in, out)")));
130 }
131 (None, scale) => scale.unwrap_or(1.0),
132 };
133 let rotate = self.rotate.unwrap_or(0.0);
134 pinch::check_motion(scale, rotate).map_err(Fail::invalid)?;
135 Ok((scale, rotate))
136 }
137}
138
139#[derive(Debug, Default, Deserialize)]
140#[serde(default, deny_unknown_fields)]
141pub struct ScrollArgs {
142 pub direction: String,
143 pub until: Option<Value>,
144 pub max: Option<u32>,
145}
146
147#[derive(Debug, Default, Deserialize)]
148#[serde(default, deny_unknown_fields)]
149pub struct TypeArgs {
150 pub text: String,
151 pub clear: bool,
152 pub submit: bool,
153}
154
155struct TypeObservation {
158 before: Option<String>,
159 value: Option<String>,
160 changed: Option<bool>,
161 warning: Option<Value>,
162}
163
164impl TypeObservation {
165 fn new(args: &TypeArgs, before: Option<String>, value: Option<String>) -> Self {
166 let changed = before.as_ref().zip(value.as_ref()).map(|(a, b)| a != b);
167 let retry_hint = t(
168 "입력이 반영되었을 수 있습니다. 자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요.",
169 "Input may have taken effect. Do not automatically retype; check with `iph read` or a screenshot.",
170 );
171 let warning = if value.is_none() {
172 Some(json!({
173 "code": "value_unreadable",
174 "message": "the input was sent but the field's value could not be read afterwards",
175 "hint": retry_hint,
176 }))
177 } else if args.text.is_empty() && value.as_deref() == Some("") {
178 None
180 } else if args.text.is_empty() {
181 Some(json!({
182 "code": "clear_not_observed",
183 "message": "the clear input was sent but the field still shows a nonempty value",
184 "hint": retry_hint,
185 }))
186 } else if changed == Some(false) {
187 Some(json!({
188 "code": "value_unchanged",
189 "message": "the field's value is unchanged; the paste may not have taken effect (identical replacement, masking or app behavior can also cause this)",
190 "hint": t(
191 "자동으로 재입력하지 말고 `iph read`나 스크린샷으로 확인하세요. 붙여넣기가 막혔을 수도 있습니다: 설정 → 앱 → 해당 앱 → 다른 앱에서 붙여넣기 → 허용. 값이 같다는 사실만으로 권한 거부를 확정할 수는 없습니다.",
192 "Do not automatically retype; check with `iph read` or a screenshot. Pasting may be blocked: Settings → Apps → this app → Paste from Other Apps → Allow. An unchanged value alone does not prove permission was denied.",
193 ),
194 }))
195 } else if value.as_deref() == Some("") {
196 Some(json!({
197 "code": "value_empty",
198 "message": "the input was sent but the field shows an empty value; the app may have cleared it",
199 "hint": retry_hint,
200 }))
201 } else if changed.is_none() {
202 Some(json!({
203 "code": "value_change_unknown",
204 "message": "the field's value was read afterwards, but its previous value was unavailable for comparison",
205 "hint": retry_hint,
206 }))
207 } else {
208 None
209 };
210 Self {
211 before,
212 value,
213 changed,
214 warning,
215 }
216 }
217
218 fn can_submit(&self) -> bool {
219 self.warning.is_none()
220 }
221
222 fn result(
223 self,
224 args: &TypeArgs,
225 field: &Element,
226 restored: bool,
227 submitted: Option<bool>,
228 screen_changed: Option<bool>,
229 submit_error: Option<String>,
230 ) -> Res {
231 let blocked_submit = args.submit && !self.can_submit();
232 let mut out = fields! {
233 "sent": true,
234 "verified": self.value.is_some(),
235 "value_before": self.before,
236 "value": self.value,
237 "value_changed": self.changed,
238 "field": brief(field),
239 "submit_sent": submitted,
240 "clipboard_restored": restored,
241 "screen_changed": screen_changed,
242 };
243 if let Some(warning) = self.warning {
244 out.insert("warning".into(), warning);
245 }
246 let fail = if let Some(e) = submit_error {
247 Some(Fail::unknown("input_failed", e))
248 } else if blocked_submit {
249 Some(Fail::unknown(
250 "type_not_verified",
251 "the input was sent, but its effect is uncertain; Return was not sent",
252 ))
253 } else {
254 None
255 };
256 if let Some(mut fail) = fail {
257 fail.hint = Some(t(
258 "입력은 이미 나갔습니다. 자동으로 재입력하거나 제출하지 말고 `iph read`나 스크린샷으로 먼저 확인하세요.",
259 "Input has already been sent. Do not automatically retype or submit; check with `iph read` or a screenshot first.",
260 ).into());
261 fail.extra = out;
262 Err(fail)
263 } else {
264 Ok(out)
265 }
266 }
267}
268
269#[derive(Debug, Deserialize)]
270#[serde(deny_unknown_fields)]
271pub struct KeyArgs {
272 pub key: String,
273}
274
275#[derive(Debug, Deserialize)]
276#[serde(deny_unknown_fields)]
277pub struct ButtonArgs {
278 pub button: String,
279}
280
281#[derive(Debug, Default, Deserialize)]
282#[serde(deny_unknown_fields)]
283pub struct LaunchArgs {
284 pub app: String,
285 #[serde(default)]
287 pub restart: bool,
288 #[serde(default)]
290 pub arguments: Vec<String>,
291 #[serde(default)]
292 pub environment: BTreeMap<String, String>,
293}
294
295impl LaunchArgs {
296 pub fn validate(&self) -> Res<()> {
297 for key in self.environment.keys() {
298 if key.is_empty() || key.contains('=') {
299 return Err(Fail::invalid(format!(
300 "environment names must be non-empty and contain no `=`: \"{key}\""
301 )));
302 }
303 }
304 Ok(())
305 }
306
307 pub fn has_launch_options(&self) -> bool {
309 !self.arguments.is_empty() || !self.environment.is_empty()
310 }
311
312 pub fn kill_existing(&self) -> bool {
313 self.restart || self.has_launch_options()
314 }
315
316 fn environment_dictionary(&self) -> Option<plist::Dictionary> {
317 (!self.environment.is_empty()).then(|| {
318 self.environment
319 .iter()
320 .map(|(k, v)| (k.clone(), plist::Value::String(v.clone())))
321 .collect()
322 })
323 }
324}
325
326#[derive(Debug, Deserialize)]
327#[serde(deny_unknown_fields)]
328pub struct ScreenshotArgs {
329 pub path: String,
330}
331
332pub fn parse_args<T: for<'de> Deserialize<'de>>(args: &Value) -> Res<T> {
333 let v = if args.is_null() { json!({}) } else { args.clone() };
334 serde_json::from_value(v).map_err(|e| Fail::invalid(e.to_string()))
335}
336
337pub fn brief(el: &Element) -> Value {
338 let mut v = json!({ "id": el.id, "role": el.role });
339 if let Some(l) = &el.label {
340 v["label"] = json!(l);
341 }
342 if let Some(val) = &el.value {
343 v["value"] = json!(val);
344 }
345 if let Some(i) = &el.identifier {
346 v["identifier"] = json!(i);
347 }
348 if let Some(f) = &el.frame {
349 v["frame"] = json!(f);
350 }
351 v
352}
353
354pub enum Target {
355 Selector(Selector),
356 Point(f64, f64),
357}
358
359fn daemon_silent(read_failed: &Fail) -> bool {
362 let locked = read_failed.extra.get("phone_locked").and_then(Value::as_bool) == Some(true);
363 read_failed.code == "ax_timeout" && !locked
364}
365
366pub fn parse_target(v: &Value) -> Res<Target> {
367 if let Some(obj) = v.as_object()
368 && let (Some(x), Some(y)) = (
369 obj.get("x").and_then(|n| n.as_f64()),
370 obj.get("y").and_then(|n| n.as_f64()),
371 )
372 && obj.len() == 2 {
373 return Ok(Target::Point(x, y));
374 }
375 if let Some(s) = v.as_str() {
376 let parts: Vec<&str> = s.split(',').map(|p| p.trim()).collect();
377 if parts.len() == 2
378 && let (Ok(x), Ok(y)) = (parts[0].parse::<f64>(), parts[1].parse::<f64>()) {
379 return Ok(Target::Point(x, y));
380 }
381 }
382 Selector::from_value(v)
383 .map(Target::Selector)
384 .map_err(|e| Fail::invalid(e.to_string()))
385}
386
387pub fn resolve_one<'a>(sel: &Selector, screen: &'a Screen) -> Res<&'a Element> {
389 let matches = dedupe_targets(sel.find(screen));
390 match matches.len() {
391 1 => Ok(matches[0]),
392 0 => Err(Fail::not_sent(
393 "not_found",
394 format!("no element matches {}", sel.describe()),
395 )
396 .hint(t(
397 "`iph read`로 현재 화면의 이름표를 확인하세요. 화면 밖에 있다면 `iph scroll down --until …`을 쓰세요.",
398 "Check the labels on screen with `iph read`. If it is off screen, use `iph scroll down --until …`.",
399 ))
400 .with("selector", sel.describe())
401 .with("screen_summary", screen.summary(12))),
402 n => Err(Fail::not_sent(
403 "ambiguous_selector",
404 format!("{n} elements match {}", sel.describe()),
405 )
406 .hint(t(
407 "role=, within=, index= 으로 하나만 남게 좁히세요.",
408 "Narrow it to one with role=, within= or index=.",
409 ))
410 .with("selector", sel.describe())
411 .with(
412 "candidates",
413 Value::Array(matches.iter().take(10).map(|e| brief(e)).collect()),
414 )),
415 }
416}
417
418fn tap_point(el: &Element, screen: &Screen) -> Res<(f64, f64)> {
420 let frame = el.frame.ok_or_else(|| {
421 Fail::not_sent("no_frame", "the element has no position on screen").with("element", brief(el))
422 })?;
423 let bounds = Rect { x: 0.0, y: 0.0, w: screen.width, h: screen.height };
424 let visible = frame.intersection(&bounds).ok_or_else(|| {
425 Fail::not_sent("off_screen", "the element is outside the screen")
426 .hint(t(
427 "`iph scroll down --until …`으로 화면 안으로 가져오세요.",
428 "Bring it on screen with `iph scroll down --until …`.",
429 ))
430 .with("element", brief(el))
431 })?;
432 let covered = |x: f64, y: f64| -> Option<&Element> {
433 let hit = screen.element_at(x, y)?;
434 if hit.id == el.id {
435 return None;
436 }
437 let inside = hit.frame.is_some_and(|h| {
440 h.x >= frame.x - 1.0
441 && h.y >= frame.y - 1.0
442 && h.x + h.w <= frame.x + frame.w + 1.0
443 && h.y + h.h <= frame.y + frame.h + 1.0
444 });
445 let encloses = hit.frame.is_some_and(|h| {
446 frame.x >= h.x - 1.0
447 && frame.y >= h.y - 1.0
448 && frame.x + frame.w <= h.x + h.w + 1.0
449 && frame.y + frame.h <= h.y + h.h + 1.0
450 });
451 (!inside && !encloses).then_some(hit)
452 };
453 let candidates = [(0.5, 0.5), (0.5, 0.25), (0.5, 0.75), (0.25, 0.5), (0.75, 0.5)];
454 let mut blocker = None;
455 for (fx, fy) in candidates {
456 let (x, y) = (visible.x + visible.w * fx, visible.y + visible.h * fy);
457 match covered(x, y) {
458 None => return Ok((x, y)),
459 Some(b) => blocker = blocker.or(Some(b)),
460 }
461 }
462 Err(Fail::not_sent("covered", "another element covers the target")
463 .hint(t(
464 "스크롤해서 가리는 요소(도구 막대 등) 밖으로 옮긴 뒤 다시 시도하세요.",
465 "Scroll so the target is clear of the covering element (a toolbar, for example) and try again.",
466 ))
467 .with("element", brief(el))
468 .with("covered_by", blocker.map(brief).unwrap_or(Value::Null)))
469}
470
471fn nfc(s: &str) -> String {
472 s.nfc().collect()
473}
474
475fn app_changed(before: &Screen, after: &Screen) -> Option<Value> {
477 let was: Vec<i64> = before.apps.iter().map(|a| a.pid).collect();
478 let now = after.apps.first()?;
479 (!was.contains(&now.pid)).then(|| json!({ "name": now.name, "pid": now.pid }))
480}
481
482impl Session {
483 fn build_opts(all: bool, raw: bool) -> BuildOptions {
484 BuildOptions { include_offscreen: all, raw }
485 }
486
487 pub async fn cmd_read(&mut self, args: ReadArgs) -> Res {
488 let fresh = match args.max_age_ms {
489 Some(ms) => Fresh::MaxAge(Duration::from_millis(ms)),
490 None => Fresh::Force,
491 };
492 let (seq, mut screen) = self
493 .screen(fresh, args.full || args.raw, Self::build_opts(args.all, args.raw))
494 .await?;
495 if args.app
496 && let Some(front) = screen.apps.first().map(|a| a.pid) {
497 screen.elements.retain(|e| e.pid == front);
498 screen.apps.retain(|a| a.pid == front);
499 }
500 let has_ids = screen.elements.iter().any(|e| e.identifier.is_some());
501 Ok(fields! {
502 "screen": {"width": screen.width, "height": screen.height},
503 "apps": screen.apps,
504 "count": screen.elements.len(),
505 "pass": if args.full || args.raw || has_ids { "full" } else { "initial" },
506 "seq": seq,
507 "elements": screen.elements,
508 })
509 }
510
511 pub async fn cmd_find(&mut self, args: FindArgs) -> Res {
512 let sel = Selector::from_value(&args.selector).map_err(|e| Fail::invalid(e.to_string()))?;
513 let (_, screen) = self
514 .screen(Fresh::Force, sel.needs_identifier(), Self::build_opts(args.all, false))
515 .await?;
516 let matches = sel.find(&screen);
517 let mut out = fields! {
518 "selector": sel.describe(),
519 "count": matches.len(),
520 "elements": matches.iter().map(|e| (*e).clone()).collect::<Vec<_>>(),
521 };
522 if sel.needs_identifier() && screen.elements.iter().all(|e| e.identifier.is_none()) {
523 out.insert(
524 "note".into(),
525 json!("this reading came without identifiers, so id= could not match; try again"),
526 );
527 }
528 Ok(out)
529 }
530
531 pub(crate) async fn observe_with(
542 &mut self,
543 since: Instant,
544 delay: Duration,
545 careful: bool,
546 ) -> Option<(u64, Screen)> {
547 tokio::time::sleep(delay).await;
548 let mut latest = self.screen(Fresh::Force, false, BuildOptions::default()).await.ok()?;
549 let lately = since.checked_sub(UNASKED_MEMORY).unwrap_or(since);
550 let chatty = self.ax.as_ref().is_some_and(|a| a.unasked_since(lately));
551 if !careful && !chatty {
552 return Some(latest);
553 }
554 for _ in 0..2 {
555 trace(|| "observe: reading again until two readings agree".into());
556 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
557 break;
558 };
559 let same = next.1.signature() == latest.1.signature();
560 latest = next;
561 if same {
562 break;
563 }
564 }
565 Some(latest)
566 }
567
568 pub(crate) async fn observe(&mut self, since: Instant, delay: Duration) -> Option<(u64, Screen)> {
569 self.observe_with(since, delay, false).await
570 }
571
572 pub(crate) async fn observe_settled(
578 &mut self,
579 since: Instant,
580 before: &[ElementSignature],
581 ) -> Option<(u64, Screen)> {
582 let mut latest = self.observe(since, AFTER_TAP).await?;
583 while latest.1.signature() == before && since.elapsed() < CHANGE_WAIT {
584 trace(|| "observe: the screen has not changed yet; reading again".into());
585 match self.screen(Fresh::Force, false, BuildOptions::default()).await {
586 Ok(next) => latest = next,
587 Err(_) => return Some(latest),
588 }
589 }
590 if latest.1.signature() == before {
591 return Some(latest);
592 }
593 while since.elapsed() < SETTLE_WAIT {
594 let Ok(next) = self.screen(Fresh::Force, false, BuildOptions::default()).await else {
595 break;
596 };
597 let same = next.1.signature() == latest.1.signature();
598 latest = next;
599 if same {
600 break;
601 }
602 trace(|| "observe: the screen is still moving; reading again".into());
603 }
604 Some(latest)
605 }
606
607 async fn locate(
610 &mut self,
611 targets: &[Target],
612 fresh: Fresh,
613 ) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
614 let need_ids = targets
615 .iter()
616 .any(|t| matches!(t, Target::Selector(s) if s.needs_identifier()));
617 let (_, screen) = self.screen(fresh, need_ids, BuildOptions::default()).await?;
618 if screen.width <= 0.0 || screen.height <= 0.0 {
619 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
620 }
621 let mut points = Vec::with_capacity(targets.len());
622 for target in targets {
623 points.push(match target {
624 Target::Selector(sel) => {
625 let el = resolve_one(sel, &screen)?;
626 let (x, y) = tap_point(el, &screen)?;
627 (x, y, Some(el.clone()))
628 }
629 Target::Point(x, y) => {
630 if *x < 0.0 || *y < 0.0 || *x > screen.width || *y > screen.height {
631 return Err(Fail::invalid(format!(
632 "point {x},{y} is outside the {}x{} screen",
633 screen.width, screen.height
634 )));
635 }
636 (*x, *y, screen.element_at(*x, *y).cloned())
637 }
638 });
639 }
640 Ok((screen, points))
641 }
642
643 async fn locate_confirmed(&mut self, targets: &[Target]) -> Option<(Screen, Vec<(f64, f64, Option<Element>)>)> {
650 let age = self.reading_age()?;
651 if age <= reuse() || age > CONFIRMED_REUSE || !targets.iter().all(|t| matches!(t, Target::Selector(_))) {
652 return None;
653 }
654 let (_, dump, _) = self.ax.as_ref()?.latest()?;
655 let located = self.locate(targets, Fresh::MaxAge(CONFIRMED_REUSE)).await.ok()?;
656 let ax = self.ax.as_mut()?;
657 for app in &dump.applications {
658 let was = dump.elements.get(app).map(|raw| crate::ax::tree::children_of(raw, false))?;
659 let now: Vec<ObjKey> = ax
660 .ask(*app, attrs::VISIBLE_CHILDREN, ASK_WAIT)
661 .await?
662 .as_array()?
663 .iter()
664 .filter_map(ObjKey::from_ka)
665 .collect();
666 if now != was {
667 trace(|| "locate: the screen has changed since the last reading".into());
668 return None;
669 }
670 }
671 for (_, _, el) in &located.1 {
672 let el = el.as_ref()?;
673 let now = ax
674 .ask(el.key?, attrs::FRAME, ASK_WAIT)
675 .await?
676 .get("NS.rectval")
677 .and_then(|v| v.as_str())
678 .and_then(Rect::parse)?;
679 let round = |r: &Rect| (r.x.round(), r.y.round(), r.w.round(), r.h.round());
680 if el.frame.as_ref().map(round) != Some(round(&now)) {
681 trace(|| "locate: a target has moved since the last reading".into());
682 return None;
683 }
684 }
685 trace(|| format!("locate: a reading {} ms old confirmed", age.as_millis()));
686 Some(located)
687 }
688
689 async fn locate_ready(&mut self, targets: &[Target]) -> Res<(Screen, Vec<(f64, f64, Option<Element>)>)> {
694 let located = match self.locate_confirmed(targets).await {
695 Some(l) => l,
696 None => self.locate(targets, Fresh::MaxAge(reuse())).await?,
697 };
698 let started = std::time::Instant::now();
699 self.ensure_input().await?;
700 if started.elapsed() < Duration::from_millis(400) {
701 return Ok(located);
702 }
703 self.locate(targets, Fresh::Force).await
704 }
705
706 pub async fn cmd_tap(&mut self, args: TapArgs) -> Res {
707 if args.double && args.long_ms.is_some() {
708 return Err(Fail::invalid("--double and --long cannot be combined"));
709 }
710 if let Some(ms) = args.long_ms
711 && !(1..=60_000).contains(&ms)
712 {
713 return Err(Fail::invalid("--long must be 1..60000 ms"));
714 }
715 let target = parse_target(&args.target)?;
716 let (screen, mut points) = match self.locate_ready(std::slice::from_ref(&target)).await {
717 Ok(located) => located,
718 Err(read_failed) => return self.tap_unread(&target, &args, read_failed).await,
719 };
720 let (x, y, element) = points.remove(0);
721 let element_json = element.as_ref().map(brief);
722
723 let (nx, ny) = (normalize(x, screen.width), normalize(y, screen.height));
724 let before = screen.signature();
725 let input_began = Instant::now();
726 let gesture = if args.double {
727 Gesture::DoubleTap { x: nx, y: ny }
728 } else {
729 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
730 };
731 self.send_gesture(gesture).await?;
732 let after = if args.no_wait {
733 self.observe(input_began, AFTER_TAP).await
734 } else {
735 self.observe_settled(input_began, &before).await
736 };
737 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
738 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
739 if app_change.as_ref().is_some_and(|c| c["name"].is_null()) {
740 self.input_suspect = true;
742 }
743
744 let mut out = fields! {
745 "sent": true,
746 "verified": changed.is_some(),
747 "screen_changed": changed.unwrap_or(false),
748 "x": (x * 10.0).round() / 10.0,
749 "y": (y * 10.0).round() / 10.0,
750 };
751 if let Some(e) = element_json {
752 out.insert("element".into(), e);
753 }
754 if changed.is_none() {
755 out.insert(
756 "note".into(),
757 json!("the tap was sent but no reading came back to compare"),
758 );
759 }
760 if let Some(change) = app_change {
761 out.insert("app_changed".into(), json!(true));
762 out.insert("app".into(), change);
763 }
764 if !args.no_wait
766 && let Some((seq, screen)) = after {
767 out.insert("screen".into(), json!({"width": screen.width, "height": screen.height}));
768 out.insert("apps".into(), json!(screen.apps));
769 out.insert("count".into(), json!(screen.elements.len()));
770 out.insert("seq".into(), json!(seq));
771 out.insert("elements".into(), json!(screen.elements));
772 }
773 Ok(out)
774 }
775
776 fn unread_size(&self, read_failed: &Fail) -> Option<(f64, f64)> {
783 if !daemon_silent(read_failed) {
784 return None;
785 }
786 self.screen_size
787 }
788
789 async fn tap_unread(&mut self, target: &Target, args: &TapArgs, read_failed: Fail) -> Res {
793 let (Target::Point(x, y), Some((w, h))) = (target, self.unread_size(&read_failed)) else {
794 return Err(read_failed);
795 };
796 let (x, y) = (*x, *y);
797 if x < 0.0 || y < 0.0 || x > w || y > h {
798 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
799 }
800 let (nx, ny) = (normalize(x, w), normalize(y, h));
801 let gesture = if args.double {
802 Gesture::DoubleTap { x: nx, y: ny }
803 } else {
804 Gesture::Tap { x: nx, y: ny, hold: Duration::from_millis(args.long_ms.unwrap_or(50)) }
805 };
806 self.send_gesture(gesture).await?;
807 Ok(fields! {
808 "sent": true,
809 "verified": false,
810 "screen_changed": false,
811 "x": (x * 10.0).round() / 10.0,
812 "y": (y * 10.0).round() / 10.0,
813 "note": UNREAD_NOTE,
814 })
815 }
816
817 pub(crate) async fn tap_sequence(&mut self, targets: &[Value]) -> Res {
822 if targets.is_empty() {
823 return Err(Fail::invalid("the tap list is empty"));
824 }
825 let parsed: Vec<Target> = targets.iter().map(parse_target).collect::<Res<_>>()?;
826 let (screen, points) = self.locate_ready(&parsed).await?;
827 let before = screen.signature();
828 let normalized: Vec<(u16, u16)> = points
829 .iter()
830 .map(|(x, y, _)| (normalize(*x, screen.width), normalize(*y, screen.height)))
831 .collect();
832 let tapped: Vec<Value> = points
833 .iter()
834 .map(|(_, _, el)| el.as_ref().map(brief).unwrap_or(Value::Null))
835 .collect();
836 let input_began = Instant::now();
837 for (i, (nx, ny)) in normalized.iter().enumerate() {
838 let tap = Gesture::Tap { x: *nx, y: *ny, hold: Duration::from_millis(50) };
839 if let Err(mut fail) = self.send_gesture(tap).await {
840 if i > 0 {
841 fail.outcome = crate::reply::Outcome::Unknown;
843 }
844 return Err(fail
845 .hint(t(
846 "입력이 일부 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
847 "Part of the input may have gone out. Do not resend; check the screen with `iph read`.",
848 ))
849 .with("taps_sent", i));
850 }
851 }
852 let after = self.observe_with(input_began, AFTER_TAP, true).await;
853 let changed = after.as_ref().map(|(_, s)| s.signature() != before);
854 let app_change = after.as_ref().and_then(|(_, s)| app_changed(&screen, s));
855 let mut out = fields! {
856 "sent": true,
857 "verified": changed.is_some(),
858 "screen_changed": changed.unwrap_or(false),
859 "taps": tapped.len(),
860 "elements": tapped,
861 };
862 if let Some(change) = app_change {
863 out.insert("app_changed".into(), json!(true));
864 out.insert("app".into(), change);
865 }
866 Ok(out)
867 }
868
869 pub(crate) async fn drag_points(
872 &mut self,
873 from: (f64, f64),
874 to: (f64, f64),
875 duration: Duration,
876 motion: Motion,
877 ) -> Res<(bool, Option<Screen>)> {
878 let (_, screen) = self
879 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
880 .await?;
881 if screen.width <= 0.0 || screen.height <= 0.0 {
882 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
883 }
884 for (x, y) in [from, to] {
885 if x < 0.0 || y < 0.0 || x > screen.width || y > screen.height {
886 return Err(Fail::invalid(format!(
887 "point {x},{y} is outside the {}x{} screen",
888 screen.width, screen.height
889 )));
890 }
891 }
892 let before = screen.signature();
893 let a = (normalize(from.0, screen.width), normalize(from.1, screen.height));
894 let b = (normalize(to.0, screen.width), normalize(to.1, screen.height));
895 let input_began = Instant::now();
896 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
897 let mut after = self.observe(input_began, AFTER_DRAG).await.map(|(_, s)| s);
898 if motion == Motion::Flick {
899 for _ in 0..3 {
901 let next = self.observe(Instant::now(), Duration::from_millis(150)).await.map(|(_, s)| s);
902 let still = match (&after, &next) {
903 (Some(a), Some(b)) => a.signature() == b.signature(),
904 _ => true,
905 };
906 after = next.or(after);
907 if still {
908 break;
909 }
910 }
911 }
912 let changed = after.as_ref().is_some_and(|s| s.signature() != before);
913 Ok((changed, after))
914 }
915
916 async fn screen_size(&mut self) -> Res<(f64, f64)> {
917 let (_, screen) = self
918 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
919 .await?;
920 Ok((screen.width, screen.height))
921 }
922
923 pub async fn cmd_swipe(&mut self, args: SwipeArgs) -> Res {
924 let read = self
925 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
926 .await;
927 let (w, h) = match &read {
928 Ok((_, screen)) => (screen.width, screen.height),
929 Err(read_failed) => self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?,
930 };
931 let (from, to) = match (&args.direction, args.from, args.to) {
932 (Some(dir), None, None) => {
933 let (cx, cy) = (w / 2.0, h / 2.0);
935 match dir.as_str() {
936 "up" => ((cx, h * 0.70), (cx, h * 0.30)),
937 "down" => ((cx, h * 0.30), (cx, h * 0.70)),
938 "left" => ((w * 0.80, cy), (w * 0.20, cy)),
939 "right" => ((w * 0.20, cy), (w * 0.80, cy)),
940 other => {
941 return Err(Fail::invalid(format!(
942 "unknown direction `{other}` (up, down, left, right)"
943 )));
944 }
945 }
946 }
947 (None, Some(f), Some(t)) => ((f[0], f[1]), (t[0], t[1])),
948 _ => return Err(Fail::invalid("give a direction, or both from and to")),
949 };
950 let duration = Duration::from_millis(args.duration_ms.unwrap_or(250).clamp(50, 10_000));
951 let motion = if args.drag { Motion::Place } else { Motion::Flick };
952 if read.is_err() {
953 for (x, y) in [from, to] {
955 if x < 0.0 || y < 0.0 || x > w || y > h {
956 return Err(Fail::invalid(format!("point {x},{y} is outside the {w}x{h} screen")));
957 }
958 }
959 let a = (normalize(from.0, w), normalize(from.1, h));
960 let b = (normalize(to.0, w), normalize(to.1, h));
961 self.send_gesture(Gesture::Drag { from: a, to: b, duration, motion }).await?;
962 return Ok(fields! {
963 "sent": true,
964 "verified": false,
965 "screen_changed": false,
966 "from": [from.0, from.1],
967 "to": [to.0, to.1],
968 "note": UNREAD_NOTE,
969 });
970 }
971 let (changed, after) = self.drag_points(from, to, duration, motion).await?;
972 Ok(fields! {
973 "sent": true,
974 "verified": after.is_some(),
975 "screen_changed": changed,
976 "from": [from.0, from.1],
977 "to": [to.0, to.1],
978 })
979 }
980
981 pub async fn cmd_pinch(&mut self, args: PinchArgs) -> Res {
986 let (scale, rotate) = args.motion()?;
987 let duration = Duration::from_millis(args.duration_ms.unwrap_or(500).clamp(100, 10_000));
988 let target = (!args.at.is_null()).then(|| parse_target(&args.at)).transpose()?;
989 let read = match &target {
990 Some(target) => self.locate_ready(std::slice::from_ref(target)).await.map(|(screen, mut points)| {
991 let (x, y, _) = points.remove(0);
992 (screen, Some((x, y)))
993 }),
994 None => self
995 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
996 .await
997 .map(|(_, screen)| (screen, None)),
998 };
999 let (size, at, before) = match &read {
1000 Ok((screen, _)) if screen.width <= 0.0 || screen.height <= 0.0 => {
1001 return Err(Fail::not_sent("no_screen_size", "the reading has no application frame"));
1002 }
1003 Ok((screen, at)) => ((screen.width, screen.height), *at, Some(screen.signature())),
1004 Err(read_failed) => {
1005 let size = self.unread_size(read_failed).ok_or_else(|| read_failed.clone())?;
1007 let at = match &target {
1008 None => None,
1009 Some(Target::Point(x, y)) => Some((*x, *y)),
1010 Some(Target::Selector(_)) => return Err(read_failed.clone()),
1011 };
1012 (size, at, None)
1013 }
1014 };
1015 let center = at.unwrap_or((size.0 / 2.0, size.1 / 2.0));
1016 let plan = pinch::plan(&pinch::Spec { screen: size, center, scale, rotate, duration }).map_err(Fail::invalid)?;
1017 let span = [(plan.span.0 * 10.0).round() / 10.0, (plan.span.1 * 10.0).round() / 10.0];
1018 let input_began = Instant::now();
1019 self.send_gesture(Gesture::Pinch(plan)).await?;
1020 let mut out = fields! {
1021 "sent": true,
1022 "center": [(center.0 * 10.0).round() / 10.0, (center.1 * 10.0).round() / 10.0],
1023 "scale": scale,
1024 "rotate": rotate,
1025 "span": span,
1026 };
1027 let Some(before) = before else {
1028 out.insert("verified".into(), json!(false));
1029 out.insert("screen_changed".into(), json!(false));
1030 out.insert("note".into(), json!(UNREAD_NOTE));
1031 return Ok(out);
1032 };
1033 let after = self.observe(input_began, AFTER_DRAG).await.map(|(_, screen)| screen);
1034 out.insert("verified".into(), json!(after.is_some()));
1035 out.insert("screen_changed".into(), json!(after.is_some_and(|s| s.signature() != before)));
1036 Ok(out)
1037 }
1038
1039 async fn scroll_step(&mut self, direction: &str) -> Res<(bool, Option<Screen>)> {
1041 let (w, h) = self.screen_size().await?;
1042 let cx = w / 2.0;
1043 let (from, to) = match direction {
1044 "down" => ((cx, h * 0.68), (cx, h * 0.32)),
1045 "up" => ((cx, h * 0.32), (cx, h * 0.68)),
1046 other => {
1047 return Err(Fail::invalid(format!("unknown direction `{other}` (up, down)")));
1048 }
1049 };
1050 self.drag_points(from, to, Duration::from_millis(500), Motion::Place)
1053 .await
1054 }
1055
1056 pub async fn cmd_scroll(&mut self, args: ScrollArgs) -> Res {
1057 if args.direction != "up" && args.direction != "down" {
1058 return Err(Fail::invalid("direction must be up or down"));
1059 }
1060 let Some(until) = &args.until else {
1061 let (changed, after) = self.scroll_step(&args.direction).await?;
1062 return Ok(fields! {
1063 "sent": true,
1064 "verified": after.is_some(),
1065 "screen_changed": changed,
1066 "scrolls": 1,
1067 });
1068 };
1069 let sel = Selector::from_value(until).map_err(|e| Fail::invalid(e.to_string()))?;
1070 let max = args.max.unwrap_or(10).clamp(1, 100);
1071 let full = sel.needs_identifier();
1072 let mut scrolls = 0u32;
1073 let (_, mut screen) = self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?;
1074 loop {
1075 let matches = dedupe_targets(sel.find(&screen));
1076 if let Some(el) = matches.iter().find(|e| tap_point(e, &screen).is_ok()) {
1078 if matches.len() > 1 {
1079 return Err(Fail::not_sent(
1080 "ambiguous_selector",
1081 format!("{} elements match {}", matches.len(), sel.describe()),
1082 )
1083 .with("candidates", Value::Array(matches.iter().map(|e| brief(e)).collect()))
1084 .with("scrolls", scrolls));
1085 }
1086 return Ok(fields! {
1087 "found": true,
1088 "scrolls": scrolls,
1089 "screen_changed": scrolls > 0,
1090 "element": brief(el),
1091 });
1092 }
1093 if scrolls >= max {
1094 return Err(Fail::new(
1095 crate::reply::Outcome::Done,
1096 "not_found",
1097 format!("{} not found after {scrolls} scrolls", sel.describe()),
1098 )
1099 .with("found", false)
1100 .with("scrolls", scrolls)
1101 .with("screen_summary", screen.summary(12)));
1102 }
1103 let (mut changed, mut after) = self.scroll_step(&args.direction).await?;
1104 scrolls += 1;
1105 if !changed {
1106 (changed, after) = self.scroll_step(&args.direction).await?;
1109 scrolls += 1;
1110 }
1111 if !changed {
1112 return Err(Fail::new(
1113 crate::reply::Outcome::Done,
1114 "not_found",
1115 format!(
1116 "{} not found; the list stopped moving after {scrolls} scrolls",
1117 sel.describe()
1118 ),
1119 )
1120 .with("found", false)
1121 .with("scrolls", scrolls)
1122 .with("screen_summary", screen.summary(12)));
1123 }
1124 screen = match (after, full) {
1125 (Some(s), false) => s,
1126 _ => self.screen(Fresh::MaxAge(reuse()), full, BuildOptions::default()).await?.1,
1127 };
1128 }
1129 }
1130
1131 pub async fn cmd_key(&mut self, args: KeyArgs) -> Res {
1132 let (usage, modifiers) = crate::keys::parse_combo(&args.key).map_err(Fail::invalid)?;
1133 let (_, screen) = self
1134 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1135 .await?;
1136 let before = Some(screen.signature());
1137 let input_began = Instant::now();
1138 let input = self.ensure_input().await?;
1139 if let Err(e) = input.key(usage, &modifiers).await {
1140 self.drop_input();
1141 return Err(Fail::unknown("input_failed", format!("{e:#}")));
1142 }
1143 let changed = match before {
1144 Some(sig) => self.observe(input_began, AFTER_TAP).await.map(|(_, sc)| sc.signature() != sig),
1145 None => None,
1146 };
1147 Ok(fields! {
1148 "sent": true,
1149 "verified": changed.is_some(),
1150 "screen_changed": changed.unwrap_or(false),
1151 "key": args.key,
1152 })
1153 }
1154
1155 pub async fn cmd_button(&mut self, args: ButtonArgs) -> Res {
1156 let (page, usage, hold) = crate::keys::button(&args.button).map_err(Fail::invalid)?;
1157 let before = match self
1159 .screen(Fresh::MaxAge(reuse()), false, BuildOptions::default())
1160 .await
1161 {
1162 Ok((_, screen)) => Some(screen.signature()),
1163 Err(read_failed) if daemon_silent(&read_failed) => None,
1164 Err(read_failed) => return Err(read_failed),
1165 };
1166 let input_began = Instant::now();
1167 self.send_gesture(Gesture::Button { page, usage, hold: Duration::from_millis(hold) }).await?;
1168 let changed = match &before {
1169 Some(sig) => self.observe(input_began, AFTER_LAUNCH).await.map(|(_, sc)| sc.signature() != *sig),
1170 None => None,
1171 };
1172 let mut out = fields! {
1173 "sent": true,
1174 "verified": changed.is_some(),
1175 "screen_changed": changed.unwrap_or(false),
1176 "button": args.button,
1177 };
1178 if before.is_none() {
1179 out.insert("note".into(), json!(UNREAD_NOTE));
1180 }
1181 Ok(out)
1182 }
1183
1184 pub async fn cmd_apps(&mut self) -> Res {
1185 let apps = self.list_apps(true).await?;
1186 let list: Vec<Value> = apps
1187 .iter()
1188 .map(|a| {
1189 json!({
1190 "bundle_id": a.bundle_identifier,
1191 "name": a.name,
1192 "version": a.version,
1193 "first_party": a.is_first_party,
1194 "removable": a.is_removable,
1195 })
1196 })
1197 .collect();
1198 Ok(fields! { "count": list.len(), "apps": list })
1199 }
1200
1201 async fn resolve_bundle(&mut self, app: &str) -> Res<String> {
1202 if app.contains('.') && !app.contains(' ') {
1205 return Ok(app.to_string());
1206 }
1207 let want = nfc(app).to_lowercase();
1208 let apps = self.list_apps(false).await?;
1209 let hits: Vec<&idevice::core_device::AppListEntry> = apps
1210 .iter()
1211 .filter(|a| nfc(&a.name).to_lowercase() == want)
1212 .collect();
1213 match hits.len() {
1214 1 => Ok(hits[0].bundle_identifier.clone()),
1215 0 => Err(Fail::not_sent("app_not_found", format!("no installed app is named \"{app}\""))
1216 .hint(t(
1217 "`iph apps`로 이름과 bundle id를 확인하세요.",
1218 "See names and bundle ids with `iph apps`.",
1219 ))),
1220 _ => Err(Fail::not_sent(
1221 "ambiguous_app",
1222 format!("{} apps are named \"{app}\"", hits.len()),
1223 )
1224 .with(
1225 "candidates",
1226 Value::Array(hits.iter().map(|a| json!(a.bundle_identifier)).collect()),
1227 )),
1228 }
1229 }
1230
1231 pub async fn cmd_launch(&mut self, args: LaunchArgs) -> Res {
1232 args.validate()?;
1233 let bundle = self.resolve_bundle(&args.app).await?;
1234 let mut svc = self.app_service().await?;
1235 let arguments: Vec<&str> = args.arguments.iter().map(String::as_str).collect();
1236 let launch_began = Instant::now();
1237 let launched = tokio::time::timeout(
1239 Duration::from_secs(30),
1240 svc.launch_application(
1241 bundle.as_str(),
1242 &arguments,
1243 args.kill_existing(),
1244 false,
1245 args.environment_dictionary(),
1246 None,
1247 None,
1248 ),
1249 )
1250 .await
1251 .map_err(|_| "no answer within 30 s".to_string())
1252 .and_then(|r| r.map_err(|e| format!("{e:?}")));
1253 let resp = match launched {
1254 Ok(r) => r,
1255 Err(msg) => {
1256 let locked = msg.to_lowercase().contains("lock");
1257 return Err(Fail::unknown("launch_failed", msg).hint(if locked {
1258 t("폰 잠금을 풀어 주세요.", "Unlock the phone.")
1259 } else {
1260 t(
1261 "bundle id가 맞는지 `iph apps`로 확인하세요.",
1262 "Check the bundle id with `iph apps`.",
1263 )
1264 }));
1265 }
1266 };
1267 let pid = resp.pid as i64;
1268 let mut foreground = false;
1270 let mut delay = AFTER_LAUNCH;
1271 for _ in 0..4 {
1272 if let Some((_, screen)) = self.observe(launch_began, delay).await
1273 && screen.apps.iter().any(|a| a.pid == pid) {
1274 foreground = true;
1275 break;
1276 }
1277 delay = Duration::from_millis(300);
1278 }
1279 let mut out = fields! {
1280 "sent": true,
1281 "verified": foreground,
1282 "foreground": foreground,
1283 "bundle_id": bundle,
1284 "pid": pid,
1285 "restart": args.kill_existing(),
1286 };
1287 if args.has_launch_options() {
1288 out.insert("restart_implied".into(), json!(!args.restart));
1289 out.insert("arguments".into(), json!(args.arguments));
1290 out.insert("environment_keys".into(), json!(args.environment.keys().collect::<Vec<_>>()));
1291 }
1292 Ok(out)
1293 }
1294
1295 pub async fn cmd_screenshot(&mut self, args: ScreenshotArgs) -> Res {
1296 let png = self.screenshot_png().await?;
1297 let path = std::path::PathBuf::from(&args.path);
1298 if let Some(dir) = path.parent().filter(|d| !d.as_os_str().is_empty()) {
1299 std::fs::create_dir_all(dir)
1300 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", dir.display())))?;
1301 }
1302 std::fs::write(&path, &png)
1303 .map_err(|e| Fail::not_sent("write_failed", format!("{}: {e}", path.display())))?;
1304 Ok(fields! { "path": args.path, "bytes": png.len() })
1305 }
1306
1307 fn focused_field(screen: &Screen) -> Option<&Element> {
1309 let mut focused = screen.elements.iter().filter(|e| e.focused);
1310 let first = focused.next()?;
1311 focused.next().is_none().then_some(first)
1312 }
1313
1314 async fn field_value(&mut self, field: &Element) -> Option<String> {
1319 let key = field.key?;
1320 let before = field.value.as_deref();
1321 let began = Instant::now();
1322 loop {
1323 let value = self.ax.as_mut()?.ask(key, attrs::VALUE, ASK_WAIT).await?;
1324 let now = crate::ax::tree::value_text(Some(&value))?;
1325 if Some(now.as_str()) != before || began.elapsed() >= PASTE_SHOWS {
1326 return Some(now);
1327 }
1328 tokio::time::sleep(Duration::from_millis(100)).await;
1329 }
1330 }
1331
1332 pub async fn cmd_type(&mut self, args: TypeArgs) -> Res {
1333 if args.text.is_empty() && !args.clear {
1334 return Err(Fail::invalid("nothing to type"));
1335 }
1336 if args.text.len() > 1024 * 1024 {
1337 return Err(Fail::invalid("text is larger than 1 MiB"));
1338 }
1339 let (_, screen) = self
1340 .screen(Fresh::Force, false, BuildOptions::default())
1341 .await?;
1342 let Some(field) = Self::focused_field(&screen).cloned() else {
1343 return Err(Fail::not_sent(
1344 "no_focused_field",
1345 "no text field has the keyboard right now",
1346 )
1347 .hint(t(
1348 "먼저 입력할 칸을 `iph tap`으로 눌러 키보드를 띄우세요.",
1349 "Tap the field first with `iph tap` so the keyboard comes up.",
1350 ))
1351 .with("screen_summary", screen.summary(12)));
1352 };
1353 trace(|| "type: focused field found".into());
1354
1355 self.ensure_input().await?;
1358 let (adapter, port) = self.service_port(PASTEBOARD_SERVICE).await?;
1359
1360 let input = self.ensure_input().await?;
1363 let (saved, attached) = tokio::join!(Clipboard::save(&adapter, port), input.keyboard_attach());
1364 trace(|| "type: clipboard saved, keyboard attached".into());
1365 if let Err(e) = attached {
1366 return Err(Fail::not_sent("keyboard_failed", format!("{e:#}")));
1368 }
1369 if !args.text.is_empty()
1370 && let Err(e) = Clipboard::set_text(&adapter, port, &args.text).await {
1371 let _ = input.keyboard_detach().await;
1372 self.invalidate_readings();
1373 return Err(Fail::not_sent("clipboard_failed", e));
1374 }
1375 trace(|| "type: clipboard set".into());
1376
1377 let cmd = vec![crate::keys::MOD_COMMAND];
1378 let mut combos: Vec<(u64, Vec<u64>)> = Vec::new();
1379 if args.clear {
1380 combos.push((crate::keys::usage('a').unwrap(), cmd.clone()));
1381 combos.push((crate::keys::KEY_BACKSPACE, vec![]));
1382 }
1383 if !args.text.is_empty() {
1384 combos.push((crate::keys::usage('v').unwrap(), cmd.clone()));
1385 }
1386 let keys_began = Instant::now();
1387 let pressed = input.keyboard_press(&combos).await;
1388 trace(|| "type: keys sent".into());
1389 if let Err(e) = pressed {
1390 let _ = input.keyboard_detach().await;
1391 self.drop_input();
1392 self.invalidate_readings();
1393 let restored = saved.restore(&adapter, port, &args.text).await;
1394 return Err(Fail::unknown("input_failed", format!("{e:#}"))
1395 .hint(t(
1396 "입력이 나갔을 수 있습니다. 다시 보내지 말고 `iph read`로 화면을 확인하세요.",
1397 "The input may have gone out. Do not resend; check the screen with `iph read`.",
1398 ))
1399 .with("clipboard_restored", restored));
1400 }
1401
1402 let restoring = saved.restore(&adapter, port, &args.text);
1409 let reading = async {
1410 tokio::time::sleep(AFTER_PASTE).await;
1411 if let Some(value) = self.field_value(&field).await {
1412 return Some(value);
1413 }
1414 let (_, screen) = self.observe(keys_began, Duration::ZERO).await?;
1415 screen.by_id(&field.id).and_then(|e| e.value.clone())
1418 };
1419 let (restored, value) = tokio::join!(restoring, reading);
1420 trace(|| format!("type: read back={} clipboard_restored={restored}", value.is_some()));
1421 let observation = TypeObservation::new(&args, field.value.clone(), value);
1422
1423 let mut submitted = Some(false);
1424 let mut submit_error = None;
1425 if args.submit && observation.can_submit() {
1426 if let Some(input) = self.input.as_mut() {
1427 match input.keyboard_press(&[(crate::keys::KEY_RETURN, vec![])]).await {
1428 Ok(()) => submitted = Some(true),
1429 Err(e) => {
1430 submitted = None;
1431 submit_error = Some(format!("{e:#}"));
1432 }
1433 }
1434 } else {
1435 submit_error = Some("the input session closed before Return could be sent".into());
1436 }
1437 }
1438 if let Some(input) = self.input.as_mut()
1439 && input.keyboard_detach().await.is_err() {
1440 self.drop_input();
1441 }
1442 self.invalidate_readings();
1445
1446 if submit_error.is_some() {
1447 self.drop_input();
1448 }
1449 let screen_changed = self
1452 .observe(keys_began, AFTER_PASTE)
1453 .await
1454 .map(|(_, after)| after.signature() != screen.signature());
1455 let mut result = observation.result(
1456 &args,
1457 &field,
1458 restored,
1459 submitted,
1460 screen_changed,
1461 submit_error,
1462 );
1463 if !restored && saved.unreadable() {
1464 let out = match &mut result {
1465 Ok(out) => out,
1466 Err(fail) => &mut fail.extra,
1467 };
1468 out.insert(
1469 "clipboard_note".into(),
1470 json!(t(
1471 "폰 클립보드에 앱에서 복사한 내용이 있었는데, 그 내용은 컴퓨터에서 읽을 수 없어(읽으면 폰 클립보드가 1분쯤 멈춤) 되돌리지 못했습니다. 입력한 글은 클립보드에서 지웠습니다.",
1472 "The phone's clipboard held something copied in an app, which cannot be read from here (reading it stalls the phone's clipboard for about a minute), so it could not be put back. The typed text was cleared from the clipboard.",
1473 )),
1474 );
1475 }
1476 result
1477 }
1478}
1479
1480const PASTEBOARD_SERVICE: &str = "com.apple.coredevice.pasteboardservice";
1481
1482const SERVICE_KINDS: [&str; 6] = [
1493 "public.utf8-plain-text",
1494 "public.plain-text",
1495 "public.text",
1496 "public.png",
1497 "public.jpeg",
1498 "public.tiff",
1499];
1500const RESOLVE_WAIT: Duration = Duration::from_secs(2);
1502
1503fn app_provided(types: &[String]) -> bool {
1506 types.iter().any(|t| !SERVICE_KINDS.contains(&t.as_str()))
1507}
1508
1509#[derive(Debug, PartialEq)]
1511enum Peek {
1512 Empty,
1513 Text(String),
1514 Image(String, Vec<u8>),
1515 AppContent,
1517 Unknown,
1519}
1520
1521struct Clipboard {
1523 before: Peek,
1524}
1525
1526impl Clipboard {
1527 async fn connect(
1530 adapter: &crate::tcp::handle::AdapterHandle,
1531 port: u16,
1532 ) -> Result<idevice::core_device::PasteboardServiceClient<Box<dyn idevice::ReadWrite>>, String> {
1533 use idevice::RsdService;
1534 let mut adapter = adapter.clone();
1535 let stream = tokio::time::timeout(Duration::from_secs(8), adapter.connect(port))
1536 .await
1537 .map_err(|_| "clipboard service: no answer".to_string())?
1538 .map_err(|e| format!("clipboard service: {e}"))?;
1539 idevice::core_device::PasteboardServiceClient::from_stream(Box::new(stream))
1540 .await
1541 .map_err(|e| format!("clipboard service: {e:?}"))
1542 }
1543
1544 async fn peek(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Peek {
1547 use idevice::core_device::{DataInclusionPolicy, PasteboardPayload};
1548 let general = idevice::core_device::GENERAL_PASTEBOARD;
1549 let Ok(mut clip) = Self::connect(adapter, port).await else {
1550 return Peek::Unknown;
1551 };
1552 let pulled = tokio::time::timeout(
1553 Duration::from_secs(8),
1554 clip.get_with_policy(general, DataInclusionPolicy::AllPromised),
1555 )
1556 .await;
1557 let Ok(Ok(snap)) = pulled else {
1558 return Peek::Unknown;
1559 };
1560 trace(|| {
1561 let kinds: Vec<String> = snap.items.iter().map(|i| i.types.join(",")).collect();
1562 format!("clipboard: kinds [{}]", kinds.join(" | "))
1563 });
1564 let Some(item) = snap.items.first() else {
1565 return Peek::Empty;
1566 };
1567 if snap.items.iter().any(|i| app_provided(&i.types)) {
1568 return Peek::AppContent;
1569 }
1570 for kind in SERVICE_KINDS {
1572 let Some(entry) = item.data.iter().find(|e| e.uti == kind) else {
1573 continue;
1574 };
1575 let bytes = match &entry.payload {
1576 PasteboardPayload::Inline(b) => Some(b.clone()),
1577 PasteboardPayload::Promised { .. } => {
1578 match tokio::time::timeout(RESOLVE_WAIT, clip.resolve(general, item.index as i64, kind)).await {
1579 Ok(Ok(b)) => b,
1580 Ok(Err(_)) => None,
1581 Err(_) => return Peek::Unknown,
1583 }
1584 }
1585 PasteboardPayload::Error(_) => None,
1586 };
1587 let Some(bytes) = bytes else { continue };
1588 return if ["public.png", "public.jpeg", "public.tiff"].contains(&kind) {
1589 Peek::Image(kind.to_string(), bytes)
1590 } else {
1591 match String::from_utf8(bytes) {
1592 Ok(text) => Peek::Text(text),
1593 Err(_) => continue,
1594 }
1595 };
1596 }
1597 Peek::Unknown
1598 }
1599
1600 async fn set_text(adapter: &crate::tcp::handle::AdapterHandle, port: u16, text: &str) -> Result<(), String> {
1601 let mut clip = Self::connect(adapter, port).await?;
1602 let general = idevice::core_device::GENERAL_PASTEBOARD;
1603 match tokio::time::timeout(Duration::from_secs(8), clip.set_text(text, general)).await {
1604 Ok(Ok(())) => Ok(()),
1605 Ok(Err(e)) => Err(format!("set clipboard: {e:?}")),
1606 Err(_) => Err("set clipboard: no answer".into()),
1607 }
1608 }
1609
1610 async fn save(adapter: &crate::tcp::handle::AdapterHandle, port: u16) -> Clipboard {
1611 Clipboard { before: Self::peek(adapter, port).await }
1612 }
1613
1614 fn unreadable(&self) -> bool {
1617 matches!(self.before, Peek::AppContent | Peek::Unknown)
1618 }
1619
1620 async fn restore(&self, adapter: &crate::tcp::handle::AdapterHandle, port: u16, typed: &str) -> bool {
1623 if typed.is_empty() {
1624 return true;
1625 }
1626 tokio::time::sleep(Duration::from_millis(150)).await;
1628 if Self::peek(adapter, port).await != Peek::Text(typed.to_string()) {
1631 return false;
1632 }
1633 let general = idevice::core_device::GENERAL_PASTEBOARD;
1634 let wrote = match &self.before {
1635 Peek::Text(text) => Self::set_text(adapter, port, text).await.is_ok(),
1636 Peek::Image(uti, bytes) => match Self::connect(adapter, port).await {
1637 Ok(mut clip) => {
1638 matches!(
1639 tokio::time::timeout(Duration::from_secs(8), clip.set_image(bytes, uti, general)).await,
1640 Ok(Ok(()))
1641 )
1642 }
1643 Err(_) => false,
1644 },
1645 Peek::Empty | Peek::AppContent | Peek::Unknown => {
1648 let _ = Self::set_text(adapter, port, "").await;
1649 return matches!(self.before, Peek::Empty);
1650 }
1651 };
1652 if !wrote {
1653 return false;
1654 }
1655 match (Self::peek(adapter, port).await, &self.before) {
1656 (Peek::Text(now), Peek::Text(old)) => &now == old,
1657 (Peek::Image(..), Peek::Image(..)) => true,
1658 _ => false,
1659 }
1660 }
1661}
1662
1663#[cfg(test)]
1664mod type_tests {
1665 use super::*;
1666 use crate::reply::envelope;
1667
1668 fn field(before: Option<&str>) -> Element {
1669 serde_json::from_value(json!({
1670 "id": "original", "role": "text_field", "value": before, "focused": true,
1671 }))
1672 .unwrap()
1673 }
1674
1675 fn result(
1676 args: &TypeArgs,
1677 before: Option<&str>,
1678 after: Option<&str>,
1679 screen: Option<bool>,
1680 ) -> Value {
1681 let field = field(before);
1682 let observed = TypeObservation::new(args, field.value.clone(), after.map(String::from));
1683 let submit = args.submit && observed.can_submit();
1684 envelope(
1685 observed.result(args, &field, true, Some(submit), screen, None),
1686 Instant::now(),
1687 )
1688 }
1689
1690 #[test]
1691 fn changed_values_are_observations_not_exact_text_matches() {
1692 for value in ["30000", "₩ 30,000", "•••••", "corrected text", " \n "] {
1693 let args = TypeArgs {
1694 text: "30000".into(),
1695 submit: true,
1696 ..Default::default()
1697 };
1698 let out = result(&args, Some(""), Some(value), Some(true));
1699 assert_eq!(out["ok"], true, "{out}");
1700 assert_eq!(out["outcome"], "done");
1701 assert_eq!(out["verified"], true);
1702 assert_eq!(out["value_before"], "");
1703 assert_eq!(out["value"], value);
1704 assert_eq!(out["value_changed"], true);
1705 assert_eq!(out["screen_changed"], true);
1706 assert_eq!(out["submit_sent"], true);
1707 assert!(out.get("warning").is_none());
1708 }
1709 }
1710
1711 #[test]
1712 fn unchanged_paste_warns_without_claiming_permission_was_denied() {
1713 let args = TypeArgs {
1714 text: "테스트 문장".into(),
1715 ..Default::default()
1716 };
1717 for value in ["", "old text", "•••••"] {
1718 let out = result(&args, Some(value), Some(value), Some(false));
1719 assert_eq!(out["ok"], true);
1720 assert_eq!(out["outcome"], "done");
1721 assert_eq!(out["sent"], true);
1722 assert_eq!(out["verified"], true, "verified still means read back");
1723 assert_eq!(out["value_changed"], false);
1724 assert_eq!(out["screen_changed"], false);
1725 assert_eq!(out["warning"]["code"], "value_unchanged");
1726 assert!(
1727 out["warning"]["message"]
1728 .as_str()
1729 .unwrap()
1730 .contains("may not")
1731 );
1732 assert!(out["warning"]["hint"].as_str().is_some());
1733 }
1734 let out = result(&args, Some(""), Some(""), Some(true));
1736 assert_eq!(out["value_changed"], false);
1737 assert_eq!(out["screen_changed"], true);
1738 assert!(out.get("warning").is_some());
1739 }
1740
1741 #[test]
1742 fn unreadable_and_uncomparable_values_do_not_become_empty_strings() {
1743 let args = TypeArgs {
1744 text: "hello".into(),
1745 ..Default::default()
1746 };
1747 let out = result(&args, Some("old"), None, None);
1748 assert_eq!(out["ok"], true);
1749 assert_eq!(out["verified"], false);
1750 assert!(out["value"].is_null());
1751 assert!(out["value_changed"].is_null());
1752 assert!(out["screen_changed"].is_null());
1753 assert_eq!(out["warning"]["code"], "value_unreadable");
1754 let out = result(&args, None, Some("hello"), Some(true));
1755 assert_eq!(out["verified"], true);
1756 assert!(out["value_before"].is_null());
1757 assert!(out["value_changed"].is_null());
1758 assert_eq!(out["warning"]["code"], "value_change_unknown");
1759 let out = result(&args, Some(""), Some("hello"), None);
1761 assert_eq!(out["verified"], true);
1762 assert!(out["screen_changed"].is_null());
1763 }
1764
1765 #[test]
1766 fn submit_is_withheld_when_the_input_effect_needs_review() {
1767 let args = TypeArgs {
1768 text: "hello".into(),
1769 submit: true,
1770 ..Default::default()
1771 };
1772 for (before, after, code) in [
1773 (Some(""), Some(""), "value_unchanged"),
1774 (Some("old"), Some("old"), "value_unchanged"),
1775 (Some("old"), None, "value_unreadable"),
1776 (None, Some("hello"), "value_change_unknown"),
1777 (Some("old"), Some(""), "value_empty"),
1778 ] {
1779 let out = result(&args, before, after, Some(false));
1780 assert_eq!(out["ok"], false, "{out}");
1781 assert_eq!(out["outcome"], "unknown");
1782 assert_eq!(out["sent"], true);
1783 assert_eq!(out["submit_sent"], false);
1784 assert_eq!(out["error"]["code"], "type_not_verified");
1785 assert_eq!(out["warning"]["code"], code);
1786 assert_eq!(out["value"], json!(after));
1787 assert_eq!(out["clipboard_restored"], true);
1788 }
1789 }
1790
1791 #[test]
1792 fn clear_and_identical_replacement_are_reported_separately() {
1793 let mut args = TypeArgs {
1794 clear: true,
1795 submit: true,
1796 ..Default::default()
1797 };
1798 for before in [Some(""), Some("old"), None] {
1799 let out = result(&args, before, Some(""), Some(false));
1800 assert_eq!(out["ok"], true);
1801 assert_eq!(out["submit_sent"], true);
1802 assert!(out.get("warning").is_none());
1803 }
1804 let out = result(&args, Some("old"), Some("old"), Some(false));
1805 assert_eq!(out["warning"]["code"], "clear_not_observed");
1806 assert_eq!(out["submit_sent"], false);
1807 args.text = "same".into();
1810 let out = result(&args, Some("same"), Some("same"), Some(false));
1811 assert_eq!(out["outcome"], "unknown");
1812 assert_eq!(out["warning"]["code"], "value_unchanged");
1813 }
1814
1815 #[test]
1816 fn failed_return_keeps_observations_and_does_not_claim_it_was_unsent() {
1817 let args = TypeArgs {
1818 text: "hello".into(),
1819 submit: true,
1820 ..Default::default()
1821 };
1822 let field = field(Some(""));
1823 let observed = TypeObservation::new(&args, field.value.clone(), Some("hello".into()));
1824 let out = envelope(
1825 observed.result(
1826 &args,
1827 &field,
1828 false,
1829 None,
1830 None,
1831 Some("write failed".into()),
1832 ),
1833 Instant::now(),
1834 );
1835 assert_eq!(out["outcome"], "unknown");
1836 assert_eq!(out["error"]["code"], "input_failed");
1837 assert!(out["submit_sent"].is_null());
1838 assert_eq!(out["value_changed"], true);
1839 assert_eq!(out["clipboard_restored"], false);
1840 }
1841}
1842
1843#[cfg(test)]
1844mod clipboard_tests {
1845 use super::app_provided;
1846
1847 fn kinds(k: &[&str]) -> Vec<String> {
1848 k.iter().map(|s| s.to_string()).collect()
1849 }
1850
1851 #[test]
1852 fn notes_copy_is_app_content() {
1853 assert!(app_provided(&kinds(&[
1855 "public.utf8-plain-text",
1856 "com.apple.notes.richtext",
1857 "com.apple.webarchive",
1858 "com.apple.flat-rtfd",
1859 "iOS rich content paste pasteboard type",
1860 "public.html",
1861 "net.daringfireball.markdown",
1862 "public.rtf",
1863 ])));
1864 }
1865
1866 #[test]
1867 fn what_iph_puts_there_is_not() {
1868 assert!(!app_provided(&kinds(&["public.utf8-plain-text", "public.plain-text", "public.text"])));
1870 assert!(!app_provided(&kinds(&["public.png"])));
1871 }
1872}
1873
1874#[cfg(test)]
1875mod launch_tests {
1876 use super::LaunchArgs;
1877 use crate::actions::parse_args;
1878 use serde_json::json;
1879
1880 fn parse(v: serde_json::Value) -> Result<LaunchArgs, crate::reply::Fail> {
1881 parse_args::<LaunchArgs>(&v)
1882 }
1883
1884 #[test]
1885 fn a_plain_launch_sends_nothing_extra() {
1886 let a = parse(json!({"app": "x"})).unwrap();
1887 assert!(!a.kill_existing());
1888 assert!(a.environment_dictionary().is_none());
1889 }
1890
1891 #[test]
1892 fn arguments_or_environment_imply_a_restart() {
1893 let a = parse(json!({"app": "x", "arguments": ["-k", "v"]})).unwrap();
1894 assert!(a.kill_existing() && !a.restart);
1895 let b = parse(json!({"app": "x", "environment": {"TZ": "UTC"}})).unwrap();
1896 assert!(b.kill_existing());
1897 let dict = b.environment_dictionary().unwrap();
1898 assert_eq!(dict.get("TZ").and_then(|v| v.as_string()), Some("UTC"));
1899 }
1900
1901 #[test]
1902 fn environment_values_may_hold_an_equals_sign() {
1903 let a = parse(json!({"app": "x", "environment": {"K": "a=b"}})).unwrap();
1904 assert!(a.validate().is_ok());
1905 }
1906
1907 #[test]
1908 fn environment_names_are_checked() {
1909 for name in ["", "A=B"] {
1910 let a = parse(json!({"app": "x", "environment": {name: "v"}})).unwrap();
1911 assert_eq!(a.validate().unwrap_err().code, "invalid_argument");
1912 }
1913 }
1914
1915 #[test]
1916 fn a_non_string_value_is_rejected_at_the_boundary() {
1917 assert_eq!(parse(json!({"app": "x", "environment": {"A": 1}})).unwrap_err().code, "invalid_argument");
1918 assert_eq!(parse(json!({"app": "x", "environment": ["A=1"]})).unwrap_err().code, "invalid_argument");
1919 assert_eq!(parse(json!({"app": "x", "arguments": "-a"})).unwrap_err().code, "invalid_argument");
1920 assert_eq!(parse(json!({"app": "x", "arguments": [1]})).unwrap_err().code, "invalid_argument");
1921 }
1922}
1923
1924#[cfg(test)]
1925mod pinch_tests {
1926 use super::*;
1927 use crate::{config::Config, reply::Outcome};
1928
1929 fn args(v: Value) -> PinchArgs {
1930 serde_json::from_value(v).unwrap()
1931 }
1932
1933 #[test]
1934 fn a_direction_means_a_scale() {
1935 assert_eq!(args(json!({"direction": "in"})).motion().unwrap(), (0.5, 0.0));
1936 assert_eq!(args(json!({"direction": "out"})).motion().unwrap(), (2.0, 0.0));
1937 assert_eq!(args(json!({"direction": "out", "rotate": -30})).motion().unwrap(), (2.0, -30.0));
1938 assert_eq!(args(json!({"scale": 1.5, "rotate": 90})).motion().unwrap(), (1.5, 90.0));
1939 assert_eq!(args(json!({"rotate": 45})).motion().unwrap(), (1.0, 45.0));
1940 }
1941
1942 #[test]
1943 fn arguments_that_cannot_make_a_gesture_are_invalid() {
1944 for bad in [
1945 json!({}),
1946 json!({"scale": 1}),
1947 json!({"direction": "out", "scale": 2}),
1948 json!({"direction": "sideways"}),
1949 json!({"scale": 0.2}),
1950 json!({"scale": 5}),
1951 json!({"rotate": 181}),
1952 json!({"direction": "in", "rotate": -181}),
1953 ] {
1954 let fail = args(bad.clone()).motion().unwrap_err();
1955 assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1956 }
1957 assert!(serde_json::from_value::<PinchArgs>(json!({"scale": 2, "zoom": 1})).is_err());
1958 }
1959
1960 #[tokio::test]
1961 async fn a_bad_request_fails_before_the_phone_is_asked_for_anything() {
1962 let mut s = Session::new("00000000-TESTONLY-NOPHONE".into(), Config::default());
1963 for bad in [json!({}), json!({"direction": "in", "scale": 2}), json!({"scale": 9})] {
1964 let fail = s.cmd_pinch(args(bad.clone())).await.unwrap_err();
1965 assert_eq!((fail.code.as_str(), fail.outcome), ("invalid_argument", Outcome::NotSent), "{bad}");
1966 }
1967 let fail = s.cmd_pinch(args(json!({"direction": "in", "at": 7}))).await.unwrap_err();
1968 assert_eq!(fail.outcome, Outcome::NotSent);
1969 }
1970}