Skip to main content

iphone_cli/device/
pinch.rs

1//! Two-finger gestures: pinch and rotate.
2//!
3//! [`plan`] turns a request into the frames to send. It works in logical
4//! points and normalizes each axis on its own at the end, so a circle stays
5//! round on a tall screen, and it does no I/O. [`play`] sends the frames and
6//! always lifts the pair afterwards, also when a frame failed.
7
8use std::time::Duration;
9
10use anyhow::Result;
11use idevice::core_device::hid::TouchscreenContact;
12
13use crate::device::input::normalize;
14
15/// The pair's contact identities. The one-finger path reports identity 2
16/// (`TOUCHSCREEN_STATE_CONTACT` is `0xC0 | 2`), so the two never collide.
17const IDENTITIES: [u8; 2] = [0, 1];
18
19pub const MIN_SCALE: f64 = 0.25;
20pub const MAX_SCALE: f64 = 4.0;
21pub const MAX_ROTATE: f64 = 180.0;
22
23/// Room kept between a finger and the screen edge.
24const MARGIN: f64 = 24.0;
25/// Closest the fingers may start or end.
26const MIN_SPAN: f64 = 40.0;
27/// The wider of the two spans, as a share of the screen's shorter side.
28const SPAN_SHARE: f64 = 0.6;
29/// Both fingers rest before they move, so the phone sees a two-finger touch begin.
30const HOLD: Duration = Duration::from_millis(40);
31/// Frames at the end pose before the lift: the last samples must say "not
32/// moving", or the content keeps coasting.
33const REST_FRAMES: u32 = 4;
34const REST_GAP: Duration = Duration::from_millis(30);
35
36/// What to do, in logical points.
37#[derive(Debug, Clone, Copy)]
38pub struct Spec {
39    pub screen: (f64, f64),
40    pub center: (f64, f64),
41    /// End span over start span: above 1 spreads the fingers, below 1 pinches.
42    pub scale: f64,
43    /// Degrees, positive clockwise on the screen.
44    pub rotate: f64,
45    /// How long the fingers move.
46    pub duration: Duration,
47}
48
49/// Both contacts at one moment, `at` after the first frame.
50#[derive(Debug, Clone, PartialEq, Eq)]
51pub(crate) struct Frame {
52    pub(crate) at: Duration,
53    pub(crate) contacts: [TouchscreenContact; 2],
54}
55
56impl Frame {
57    pub(crate) fn touching(&self) -> bool {
58        self.contacts[0].touching
59    }
60}
61
62#[derive(Debug, Clone)]
63pub struct Plan {
64    /// Down, hold, the moving frames, the rest frames, and the lift last.
65    pub(crate) frames: Vec<Frame>,
66    pub center: (f64, f64),
67    /// The distance between the fingers at the start and at the end, in points.
68    pub span: (f64, f64),
69}
70
71/// Whether the motion asks for anything the gesture can do.
72pub fn check_motion(scale: f64, rotate: f64) -> Result<(), String> {
73    if !(MIN_SCALE..=MAX_SCALE).contains(&scale) {
74        return Err(format!("scale must be {MIN_SCALE}..{MAX_SCALE}"));
75    }
76    if !(-MAX_ROTATE..=MAX_ROTATE).contains(&rotate) {
77        return Err(format!("rotate must be -{MAX_ROTATE}..{MAX_ROTATE} degrees"));
78    }
79    if scale == 1.0 && rotate == 0.0 {
80        return Err("give a direction, a scale other than 1, or a rotation".into());
81    }
82    Ok(())
83}
84
85fn smoothstep(t: f64) -> f64 {
86    t * t * (3.0 - 2.0 * t)
87}
88
89/// The frames for `spec`, or why there is no room for them. Nothing has been
90/// sent at that point. When the fingers would leave the margin the whole
91/// motion is made smaller, never clamped point by point, so the ratio between
92/// the two spans stays what was asked for.
93pub fn plan(spec: &Spec) -> Result<Plan, String> {
94    let Spec { screen: (w, h), center: (cx, cy), scale, rotate, duration } = *spec;
95    check_motion(scale, rotate)?;
96    if !(w > 0.0 && h > 0.0) {
97        return Err("the screen has no size".into());
98    }
99    let room = (cx.min(w - cx) - MARGIN, cy.min(h - cy) - MARGIN);
100    if !(room.0 > 0.0 && room.1 > 0.0) {
101        return Err(format!(
102            "{cx},{cy} is too close to the edge of the {w}x{h} screen; both fingers need {MARGIN} pt of room"
103        ));
104    }
105    let base = SPAN_SHARE * w.min(h);
106    let (mut from, mut to) = if scale >= 1.0 { (base / scale, base) } else { (base, base * scale) };
107    let angle = rotate.to_radians();
108    let steps = (duration.as_millis() / 16).clamp(4, 240) as u32;
109    let eased = |i: u32| smoothstep(i as f64 / steps as f64);
110    let offset = |from: f64, to: f64, e: f64| {
111        let (r, a) = ((from + (to - from) * e) / 2.0, angle * e);
112        (r * a.cos(), r * a.sin())
113    };
114
115    let fit = (0..=steps).fold(1.0_f64, |fit, i| {
116        let (ox, oy) = offset(from, to, eased(i));
117        let fit = if ox.abs() > f64::EPSILON { fit.min(room.0 / ox.abs()) } else { fit };
118        if oy.abs() > f64::EPSILON { fit.min(room.1 / oy.abs()) } else { fit }
119    });
120    (from, to) = (from * fit, to * fit);
121    if from.min(to) < MIN_SPAN {
122        return Err(format!(
123            "no room for a {scale}x pinch at {cx},{cy}: the fingers would be {:.0} pt apart, under {MIN_SPAN}",
124            from.min(to)
125        ));
126    }
127
128    let frame = |at: Duration, e: f64, touching: bool| {
129        let (ox, oy) = offset(from, to, e);
130        let contact = |identity: u8, sign: f64| TouchscreenContact {
131            identity,
132            touching,
133            x: normalize(cx + sign * ox, w),
134            y: normalize(cy + sign * oy, h),
135        };
136        Frame { at, contacts: [contact(IDENTITIES[0], 1.0), contact(IDENTITIES[1], -1.0)] }
137    };
138    let moved_until = HOLD + duration;
139    let mut frames = Vec::with_capacity(steps as usize + 2 + REST_FRAMES as usize + 1);
140    frames.push(frame(Duration::ZERO, 0.0, true));
141    frames.push(frame(HOLD, 0.0, true));
142    for i in 1..=steps {
143        frames.push(frame(HOLD + duration.mul_f64(i as f64 / steps as f64), eased(i), true));
144    }
145    for j in 1..=REST_FRAMES {
146        frames.push(frame(moved_until + REST_GAP * j, 1.0, true));
147    }
148    frames.push(frame(moved_until + REST_GAP * REST_FRAMES, 1.0, false));
149    Ok(Plan { frames, center: (cx, cy), span: (from, to) })
150}
151
152/// `pair` lifted where it stands.
153pub(crate) fn lifted(pair: &[TouchscreenContact; 2]) -> [TouchscreenContact; 2] {
154    pair.map(|c| TouchscreenContact { touching: false, ..c })
155}
156
157/// Send `plan` frame by frame through `send`. Once a frame may have gone out
158/// the pair is lifted, also when a later frame failed, and `held` says so
159/// until the lift went through: a gesture dropped halfway still leaves the
160/// pair for [`InputSession::close`](crate::device::input::InputSession::close)
161/// to lift.
162pub(crate) async fn play(
163    plan: &Plan,
164    held: &mut Option<[TouchscreenContact; 2]>,
165    mut send: impl AsyncFnMut(&[TouchscreenContact]) -> Result<()>,
166) -> Result<()> {
167    let start = tokio::time::Instant::now();
168    let mut result = Ok(());
169    for frame in &plan.frames {
170        tokio::time::sleep_until(start + frame.at).await;
171        if frame.touching() {
172            *held = Some(frame.contacts);
173        }
174        if let Err(e) = send(&frame.contacts).await {
175            result = Err(e);
176            break;
177        }
178        if !frame.touching() {
179            *held = None;
180        }
181    }
182    if let Some(pair) = *held {
183        let lift = send(&lifted(&pair)).await;
184        if lift.is_ok() {
185            *held = None;
186        }
187        result = result.and(lift);
188    }
189    result
190}
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195    use anyhow::anyhow;
196
197    const SCREEN: (f64, f64) = (402.0, 874.0);
198
199    fn spec(scale: f64, rotate: f64) -> Spec {
200        Spec {
201            screen: SCREEN,
202            center: (SCREEN.0 / 2.0, SCREEN.1 / 2.0),
203            scale,
204            rotate,
205            duration: Duration::from_millis(500),
206        }
207    }
208
209    fn point(c: &TouchscreenContact) -> (f64, f64) {
210        (c.x as f64 / 65535.0 * SCREEN.0, c.y as f64 / 65535.0 * SCREEN.1)
211    }
212
213    fn span(f: &Frame) -> f64 {
214        let (a, b) = (point(&f.contacts[0]), point(&f.contacts[1]));
215        (a.0 - b.0).hypot(a.1 - b.1)
216    }
217
218    fn angle(f: &Frame) -> f64 {
219        let (a, b) = (point(&f.contacts[0]), point(&f.contacts[1]));
220        (a.1 - b.1).atan2(a.0 - b.0).to_degrees()
221    }
222
223    fn turn(degrees: f64) -> f64 {
224        (degrees + 180.0).rem_euclid(360.0) - 180.0
225    }
226
227    fn touching(plan: &Plan) -> impl Iterator<Item = &Frame> {
228        plan.frames.iter().filter(|f| f.touching())
229    }
230
231    #[test]
232    fn the_end_span_is_the_requested_scale_times_the_start_span() {
233        for scale in [0.25, 0.5, 0.8, 1.5, 2.0, 4.0] {
234            let plan = plan(&spec(scale, 0.0)).unwrap();
235            let (first, last) = (plan.frames.first().unwrap(), touching(&plan).last().unwrap());
236            assert!((span(last) - scale * span(first)).abs() < 1.0, "scale {scale}");
237            assert!((plan.span.1 - scale * plan.span.0).abs() < 1e-9, "scale {scale}");
238        }
239    }
240
241    #[test]
242    fn the_fingers_stay_centred_on_the_target_in_every_frame() {
243        let target = (120.0, 300.0);
244        let plan = plan(&Spec { center: target, ..spec(2.0, 60.0) }).unwrap();
245        for f in &plan.frames {
246            let (a, b) = (point(&f.contacts[0]), point(&f.contacts[1]));
247            let mid = ((a.0 + b.0) / 2.0, (a.1 + b.1) / 2.0);
248            assert!((mid.0 - target.0).abs() < 0.05 && (mid.1 - target.1).abs() < 0.05, "{mid:?}");
249        }
250    }
251
252    #[test]
253    fn the_angle_turns_by_exactly_the_rotation() {
254        for rotate in [90.0, -45.0, 150.0, 180.0, -180.0, 10.0] {
255            let plan = plan(&spec(1.0, rotate)).unwrap();
256            let turned = angle(touching(&plan).last().unwrap()) - angle(&plan.frames[0]);
257            assert!(turn(turned - rotate).abs() < 0.1, "rotate {rotate} turned {turned}");
258        }
259    }
260
261    #[test]
262    fn a_positive_rotation_is_clockwise_with_y_pointing_down() {
263        let plan = plan(&spec(1.0, 90.0)).unwrap();
264        let centre = (SCREEN.0 / 2.0, SCREEN.1 / 2.0);
265        let (start, end) = (point(&plan.frames[0].contacts[0]), point(&touching(&plan).last().unwrap().contacts[0]));
266        assert!(start.0 > centre.0 && (start.1 - centre.1).abs() < 0.1, "starts right of the centre: {start:?}");
267        assert!(end.1 > centre.1 && (end.0 - centre.0).abs() < 0.1, "ends below the centre: {end:?}");
268    }
269
270    #[test]
271    fn a_circle_stays_round_on_a_tall_screen() {
272        let plan = plan(&spec(1.0, 180.0)).unwrap();
273        for f in touching(&plan) {
274            assert!((span(f) - plan.span.0).abs() < 0.05, "{} vs {}", span(f), plan.span.0);
275        }
276    }
277
278    #[test]
279    fn the_frames_run_forward_in_time_and_end_with_a_lift_of_both_contacts() {
280        let plan = plan(&spec(2.0, 0.0)).unwrap();
281        assert!(plan.frames.windows(2).all(|w| w[0].at <= w[1].at));
282        assert_eq!(plan.frames[0].at, Duration::ZERO);
283        let moving = &plan.frames[2..plan.frames.len() - 1 - REST_FRAMES as usize];
284        assert!(moving.iter().all(|f| f.at > HOLD && f.at <= HOLD + Duration::from_millis(500)));
285        assert_eq!(plan.frames.last().unwrap().at, HOLD + Duration::from_millis(500) + REST_GAP * REST_FRAMES);
286        let (identities, flags): (Vec<u8>, Vec<bool>) = plan
287            .frames
288            .last()
289            .unwrap()
290            .contacts
291            .iter()
292            .map(|c| (c.identity, c.touching))
293            .unzip();
294        assert_eq!((identities, flags), (vec![0, 1], vec![false, false]));
295        assert!(touching(&plan).all(|f| f.contacts.map(|c| c.identity) == [0, 1]));
296        let down = &plan.frames[0].contacts;
297        let lift = &plan.frames.last().unwrap().contacts;
298        assert_eq!(down.map(|c| c.identity), lift.map(|c| c.identity));
299    }
300
301    #[test]
302    fn a_target_near_an_edge_shrinks_the_motion_and_keeps_every_frame_inside_the_margin() {
303        let plan = plan(&Spec { center: (90.0, 437.0), ..spec(2.0, 0.0) }).unwrap();
304        let (open, tight) = (self::plan(&spec(2.0, 0.0)).unwrap(), &plan);
305        assert!(tight.span.1 < open.span.1);
306        assert!((tight.span.1 / tight.span.0 - 2.0).abs() < 1e-9);
307        for f in &plan.frames {
308            for c in &f.contacts {
309                let (x, y) = point(c);
310                assert!(x >= MARGIN - 0.05 && x <= SCREEN.0 - MARGIN + 0.05, "x {x}");
311                assert!(y >= MARGIN - 0.05 && y <= SCREEN.1 - MARGIN + 0.05, "y {y}");
312            }
313        }
314    }
315
316    #[test]
317    fn a_request_that_cannot_fit_is_an_error_before_anything_is_sent() {
318        for center in [(5.0, 5.0), (400.0, 437.0), (201.0, 10.0), (30.0, 437.0)] {
319            assert!(plan(&Spec { center, ..spec(4.0, 0.0) }).is_err(), "{center:?}");
320        }
321        assert!(plan(&Spec { center: (60.0, 437.0), ..spec(2.0, 0.0) }).is_err(), "36 pt apart is under the minimum");
322        assert!(plan(&Spec { screen: (0.0, 874.0), ..spec(2.0, 0.0) }).is_err());
323    }
324
325    #[test]
326    fn a_motion_outside_the_limits_or_with_nothing_to_do_is_rejected() {
327        for (scale, rotate) in [(1.0, 0.0), (0.1, 0.0), (5.0, 0.0), (2.0, 181.0), (2.0, -200.0), (f64::NAN, 0.0), (1.0, f64::NAN)] {
328            assert!(check_motion(scale, rotate).is_err(), "{scale} {rotate}");
329            assert!(plan(&spec(scale, rotate)).is_err(), "{scale} {rotate}");
330        }
331        for (scale, rotate) in [(0.25, 0.0), (4.0, 0.0), (1.0, 180.0), (1.0, -180.0), (2.0, 30.0)] {
332            assert!(check_motion(scale, rotate).is_ok(), "{scale} {rotate}");
333        }
334    }
335
336    #[test]
337    fn the_longest_gesture_is_capped_at_240_steps() {
338        let plan = plan(&Spec { duration: Duration::from_secs(10), ..spec(2.0, 0.0) }).unwrap();
339        assert_eq!(plan.frames.len(), 240 + 2 + REST_FRAMES as usize + 1);
340    }
341
342    #[tokio::test(start_paused = true)]
343    async fn the_frames_go_out_in_order_and_the_pair_is_not_held_afterwards() {
344        let plan = plan(&spec(2.0, 0.0)).unwrap();
345        let (mut sent, mut held) = (Vec::new(), None);
346        play(&plan, &mut held, async |c: &[TouchscreenContact]| {
347            sent.push(c.to_vec());
348            Ok(())
349        })
350        .await
351        .unwrap();
352        assert_eq!(sent.len(), plan.frames.len());
353        assert!(held.is_none());
354        assert!(sent.last().unwrap().iter().all(|c| !c.touching));
355    }
356
357    #[tokio::test(start_paused = true)]
358    async fn a_failure_on_a_frame_still_sends_the_lift_where_the_pair_was() {
359        let plan = plan(&spec(2.0, 0.0)).unwrap();
360        for failing in [0, 1, 5, plan.frames.len() - 2] {
361            let (mut sent, mut held) = (Vec::new(), None);
362            let result = play(&plan, &mut held, async |c: &[TouchscreenContact]| {
363                sent.push(c.to_vec());
364                if sent.len() == failing + 1 { Err(anyhow!("frame {failing} failed")) } else { Ok(()) }
365            })
366            .await;
367            assert_eq!(result.unwrap_err().to_string(), format!("frame {failing} failed"));
368            assert_eq!(sent.len(), failing + 2, "the failed frame, then the lift");
369            let lift = sent.last().unwrap();
370            assert_eq!(lift.iter().map(|c| (c.identity, c.touching)).collect::<Vec<_>>(), [(0, false), (1, false)]);
371            let before = &plan.frames[failing].contacts;
372            assert_eq!(lift.iter().map(|c| (c.x, c.y)).collect::<Vec<_>>(), before.map(|c| (c.x, c.y)));
373            assert!(held.is_none());
374        }
375    }
376
377    #[tokio::test(start_paused = true)]
378    async fn a_lift_that_fails_leaves_the_pair_held_for_close_to_lift() {
379        let plan = plan(&spec(2.0, 0.0)).unwrap();
380        let (mut sent, mut held) = (Vec::new(), None);
381        let result = play(&plan, &mut held, async |c: &[TouchscreenContact]| {
382            sent.push(c.to_vec());
383            if c[0].touching && sent.len() < 3 { Ok(()) } else { Err(anyhow!("down")) }
384        })
385        .await;
386        assert!(result.is_err());
387        assert!(held.is_some(), "the lift went nowhere, so the pair is still down");
388    }
389
390    #[tokio::test(start_paused = true)]
391    async fn the_frames_are_paced_by_their_times() {
392        let plan = plan(&spec(2.0, 0.0)).unwrap();
393        let (start, mut held) = (tokio::time::Instant::now(), None);
394        let mut times = Vec::new();
395        play(&plan, &mut held, async |_: &[TouchscreenContact]| {
396            times.push(start.elapsed());
397            Ok(())
398        })
399        .await
400        .unwrap();
401        assert_eq!(times.len(), plan.frames.len());
402        for (ran, f) in times.iter().zip(&plan.frames) {
403            assert!(ran.abs_diff(f.at) <= Duration::from_millis(1), "{ran:?} vs {:?}", f.at);
404        }
405    }
406}