Skip to main content

iphone_cli/device/
input.rs

1//! Touch, keyboard and hardware-button input.
2//!
3//! The phone drops every synthetic HID event unless a displayservice media
4//! stream is active, so an [`InputSession`] first starts a screen-sharing
5//! stream (audio + video, the way Xcode's Devices window does), throws the RTP
6//! away, and then talks to the universal HID service.
7//!
8//! The media-session bring-up is adapted from the `coredevice_stream` example
9//! tool in jkcoxson/idevice (MIT, see LICENSE.idevice).
10
11use std::{
12    sync::{
13        Arc,
14        atomic::{AtomicU64, Ordering},
15    },
16    time::{Duration, Instant},
17};
18
19use anyhow::{Result, anyhow};
20use idevice::{
21    ReadWrite, RsdService,
22    core_device::{
23        CallInfoBlob, DisplayServiceClient, build_screen_audio_offer, build_screen_video_offer,
24        build_start_audio_parameters, build_start_video_parameters,
25        hid::{
26            ButtonState, IndigoHidClient, KeyboardUsage, MainKeyboardService,
27            TOUCHSCREEN_STATE_CONTACT, TOUCHSCREEN_STATE_RELEASE, TouchscreenContact,
28            UniversalHidServiceClient,
29        },
30    },
31    rsd::RsdHandshake,
32};
33
34use crate::{
35    device::pinch,
36    tcp::handle::{AdapterHandle, UdpSocketHandle},
37};
38
39/// What the controller says it supports. Xcode's device window sends 140.
40const CLIENT_SUPPORTED_FEATURES: u64 = 140;
41
42/// Longest wait for the phone to make a new connection's HID devices.
43const DEVICES_READY: Duration = Duration::from_secs(5);
44/// Time between the phone having made its HID devices and the first touch.
45/// The phone makes a connection's devices one after another and the system
46/// then opens them; a touch that comes before the touchscreen is open is held
47/// and delivered with the rest all at once, too fast for an edge swipe
48/// (2026-11-12, dtuhidd's log). The Mac cannot see the opening, so this is
49/// measured: on battery over Wi-Fi with a new session for every command, 0 of
50/// 40 taps and 0 of 40 swipes back lost (it was 2.5 s and some 9% lost while
51/// every session made three connections). See how-it-works ยง3.
52const WARMUP: Duration = Duration::from_millis(300);
53/// Keep the touch connection quiet for this long after a release; closing or
54/// reusing it earlier makes the phone drop some taps. The wait happens before
55/// the next input (and before closing), not after each one, so the caller can
56/// already ask for the new screen meanwhile.
57const SETTLE: Duration = Duration::from_millis(300);
58/// Shortest pause between one release and the next touch on the open
59/// connection, so two taps are not read as a double tap or a drag.
60const TOUCH_GAP: Duration = Duration::from_millis(150);
61
62/// Longest extra wait at the start of a session for the media stream's
63/// first packets.
64const FIRST_PACKET: Duration = Duration::from_secs(3);
65
66/// Time between registering the temporary keyboard and its first key.
67const KEYBOARD_ATTACH: Duration = Duration::from_millis(700);
68
69/// `WARMUP`, or `IPH_INPUT_WARMUP_MS` when set (for tuning).
70fn warmup() -> Duration {
71    std::env::var("IPH_INPUT_WARMUP_MS")
72        .ok()
73        .and_then(|v| v.parse().ok())
74        .map(Duration::from_millis)
75        .unwrap_or(WARMUP)
76}
77
78/// Wait until the phone has made this connection's HID devices: when it
79/// answers the list of them. For the trace: when, or why not.
80async fn devices_ready<R: ReadWrite>(client: &mut UniversalHidServiceClient<R>) -> String {
81    let began = now_ms();
82    match tokio::time::timeout(DEVICES_READY, client.list_connected_services()).await {
83        Ok(Ok(_)) => format!("{} ms", now_ms().saturating_sub(began)),
84        Ok(Err(e)) => format!("error {e:?}"),
85        Err(_) => "timed out".to_string(),
86    }
87}
88
89fn now_ms() -> u64 {
90    std::time::SystemTime::now()
91        .duration_since(std::time::UNIX_EPOCH)
92        .map(|d| d.as_millis() as u64)
93        .unwrap_or(0)
94}
95
96struct Drain {
97    last_packet_ms: Arc<AtomicU64>,
98    /// When each socket (audio, video) got its first packet; 0 = not yet.
99    first_packet_ms: Vec<Arc<AtomicU64>>,
100    tasks: Vec<tokio::task::JoinHandle<()>>,
101}
102
103impl Drain {
104    fn start(sockets: Vec<UdpSocketHandle>) -> Self {
105        let last_packet_ms = Arc::new(AtomicU64::new(0));
106        let first_packet_ms: Vec<Arc<AtomicU64>> =
107            sockets.iter().map(|_| Arc::new(AtomicU64::new(0))).collect();
108        // Bytes and packets each stream has delivered (audio, then video when
109        // it is on), for the trace: the media crosses the same tunnel as
110        // everything else.
111        let counts: Vec<Arc<(AtomicU64, AtomicU64)>> =
112            sockets.iter().map(|_| Arc::new((AtomicU64::new(0), AtomicU64::new(0)))).collect();
113        let mut tasks: Vec<tokio::task::JoinHandle<()>> = sockets
114            .into_iter()
115            .zip(first_packet_ms.iter().cloned())
116            .zip(counts.iter().cloned())
117            .map(|((s, first), count)| {
118                let last = last_packet_ms.clone();
119                tokio::spawn(async move {
120                    // The media itself is not needed; only that it keeps coming.
121                    while let Ok(packet) = s.recv().await {
122                        let now = now_ms();
123                        last.store(now, Ordering::Relaxed);
124                        let _ = first.compare_exchange(0, now, Ordering::Relaxed, Ordering::Relaxed);
125                        count.0.fetch_add(packet.data.len() as u64, Ordering::Relaxed);
126                        count.1.fetch_add(1, Ordering::Relaxed);
127                    }
128                })
129            })
130            .collect();
131        if std::env::var("IPH_TRACE").is_ok_and(|v| v == "1") {
132            tasks.push(tokio::spawn(async move {
133                let mut before: Vec<(u64, u64)> = counts.iter().map(|_| (0, 0)).collect();
134                loop {
135                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
136                    let now: Vec<(u64, u64)> = counts
137                        .iter()
138                        .map(|c| (c.0.load(Ordering::Relaxed), c.1.load(Ordering::Relaxed)))
139                        .collect();
140                    let rates: Vec<String> = now
141                        .iter()
142                        .zip(&before)
143                        .map(|(n, b)| format!("{} B/{} pkts", n.0 - b.0, n.1 - b.1))
144                        .collect();
145                    crate::device::ax::trace(|| format!("media in the last second: {}", rates.join(", ")));
146                    before = now;
147                }
148            }));
149        }
150        Self {
151            last_packet_ms,
152            first_packet_ms,
153            tasks,
154        }
155    }
156
157    /// Whether every stream has delivered something.
158    fn flowing(&self) -> bool {
159        self.first_packet_ms.iter().all(|f| f.load(Ordering::Relaxed) > 0)
160    }
161}
162
163impl Drop for Drain {
164    fn drop(&mut self) {
165        for t in &self.tasks {
166            t.abort();
167        }
168    }
169}
170
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Motion {
173    Flick,
174    Place,
175}
176
177pub struct InputSession {
178    display: DisplayServiceClient<Box<dyn ReadWrite>>,
179    hid: UniversalHidServiceClient<Box<dyn ReadWrite>>,
180    /// A second connection for the keyboard, so its request/response traffic
181    /// stays off the touch connection. Opened on the first key.
182    keyboard: Option<UniversalHidServiceClient<Box<dyn ReadWrite>>>,
183    /// The temporary keyboard while it is registered.
184    board: Option<MainKeyboardService>,
185    /// For the hardware buttons. Opened on the first button.
186    indigo: Option<IndigoHidClient<Box<dyn ReadWrite>>>,
187    /// To open those two later.
188    adapter: AdapterHandle,
189    rsd: RsdHandshake,
190    drain: Drain,
191    pub started: Instant,
192    last_release: Option<Instant>,
193    /// Touch and button reports written so far.
194    reports: u64,
195    /// The two fingers of a pinch while they may be down on the phone.
196    pair_held: Option<[TouchscreenContact; 2]>,
197}
198
199/// Normalized HID coordinate (0..=65535) for a point in logical screen points.
200pub fn normalize(v: f64, extent: f64) -> u16 {
201    if extent <= 0.0 {
202        return 0;
203    }
204    ((v / extent) * 65535.0).round().clamp(0.0, 65535.0) as u16
205}
206
207impl InputSession {
208    /// `video`: also start the screen video stream. The audio stream alone
209    /// sets up the screen-sharing session that makes touches count
210    /// (2026-11-11: taps, drags, long presses, keys and buttons all work
211    /// without the video; 24 of 24 first drags on a new session either way).
212    /// The video is up to 6 Mbit/s while the screen moves. Over Wi-Fi the
213    /// tunnel carried a seventh of that, and everything else queued behind it.
214    pub async fn open(adapter: &mut AdapterHandle, rsd: &mut RsdHandshake, video: bool) -> Result<Self> {
215        let began = Instant::now();
216        let began_ms = now_ms();
217        let mut display = DisplayServiceClient::connect_rsd(adapter, rsd)
218            .await
219            .map_err(|e| anyhow!("display service: {e:?}"))?;
220
221        let audio_udp = adapter.bind_udp(0).await?;
222        let video_udp = adapter.bind_udp(0).await?;
223        let receiver_ip = adapter.host_ip().to_string();
224        let sender_ip = adapter.peer_ip().to_string();
225
226        let call_info = CallInfoBlob {
227            call_id: 0,
228            client_version: 1,
229            device_type: "Mac17,7".into(),
230            framework_version: "2205.3.1".into(),
231            os_version: "25F71".into(),
232            device_name: None,
233            audio_device_uid: None,
234        };
235        let session_id = uuid::Uuid::new_v4();
236
237        let audio_call = uuid::Uuid::new_v4().to_string().to_uppercase();
238        let audio_offer = build_screen_audio_offer(&audio_call, &call_info)
239            .map_err(|e| anyhow!("audio offer: {e:?}"))?;
240        let audio = display
241            .start_media_stream(build_start_audio_parameters(
242                &receiver_ip,
243                audio_udp.local_port(),
244                &sender_ip,
245                50000,
246                audio_offer,
247                CLIENT_SUPPORTED_FEATURES,
248                session_id,
249            ))
250            .await;
251        let mut streams = Vec::new();
252        let mut video = video;
253        match audio {
254            Ok(_) => streams.push(audio_udp),
255            // CoreDevice 9022 "The device microphone or camera is currently
256            // in use" (a call, a recording): the phone refuses the audio
257            // stream, so the video stream alone sets up the session.
258            Err(e) if format!("{e:?}").contains("Integer(9022)") => video = true,
259            Err(e) => return Err(anyhow!("start audio stream: {e:?}")),
260        }
261        if video {
262            let video_call = uuid::Uuid::new_v4().to_string().to_uppercase();
263            let ssrc = uuid::Uuid::new_v4().as_u128() as u32;
264            let video_offer = build_screen_video_offer(&video_call, &call_info, ssrc)
265                .map_err(|e| anyhow!("video offer: {e:?}"))?;
266            display
267                .start_media_stream(build_start_video_parameters(
268                    &receiver_ip,
269                    video_udp.local_port(),
270                    &sender_ip,
271                    50001,
272                    video_offer,
273                    CLIENT_SUPPORTED_FEATURES,
274                    1,
275                    session_id,
276                ))
277                .await
278                .map_err(|e| anyhow!("start video stream: {e:?}"))?;
279            streams.push(video_udp);
280        }
281
282        let drain = Drain::start(streams);
283
284        // Only the touch connection now: every HID connection makes its own
285        // five devices on the phone (a touchscreen, a keyboard, buttons...),
286        // one after another, and a touch waits until the phone has opened
287        // them. The keyboard and the buttons connect when first used.
288        let mut hid = UniversalHidServiceClient::connect_rsd(adapter, rsd)
289            .await
290            .map_err(|e| anyhow!("hid service: {e:?}"))?;
291        let hid_ready = devices_ready(&mut hid).await;
292        // A "finger up" with no finger down does nothing on the phone, but it
293        // is a first report, and the first report on a new connection is
294        // sometimes lost, taking a whole gesture with it. Measured with a new
295        // session for every scroll: 7 of 40 first drags lost without this,
296        // 0 of 60 with it (iOS 27.0.1). `IPH_INPUT_PRIME=0` turns it off.
297        let prime = std::env::var("IPH_INPUT_PRIME").map(|v| v != "0").unwrap_or(true);
298        if prime {
299            let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
300        }
301        tokio::time::sleep(warmup()).await;
302        // Touches only count while the phone is sending the stream. It has
303        // usually begun by now; when it has not, wait for it a little.
304        let waiting = Instant::now();
305        while !drain.flowing() && waiting.elapsed() < FIRST_PACKET {
306            tokio::time::sleep(Duration::from_millis(50)).await;
307        }
308        crate::device::ax::trace(|| {
309            let firsts: Vec<String> = drain
310                .first_packet_ms
311                .iter()
312                .map(|f| match f.load(Ordering::Relaxed) {
313                    0 => "none".to_string(),
314                    at => format!("{} ms", at.saturating_sub(began_ms)),
315                })
316                .collect();
317            format!(
318                "input: session open after {} ms, devices ready at {hid_ready}, first audio/video packet at {}",
319                began.elapsed().as_millis(),
320                firsts.join(" / ")
321            )
322        });
323        if prime {
324            let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
325            tokio::time::sleep(Duration::from_millis(60)).await;
326        }
327
328        Ok(Self {
329            display,
330            hid,
331            keyboard: None,
332            board: None,
333            indigo: None,
334            adapter: adapter.clone(),
335            rsd: rsd.clone(),
336            drain,
337            started: Instant::now(),
338            last_release: None,
339            reports: 0,
340            pair_held: None,
341        })
342    }
343
344    /// Milliseconds since the phone last sent a media packet, if it ever did.
345    pub fn media_idle_ms(&self) -> Option<u64> {
346        let last = self.drain.last_packet_ms.load(Ordering::Relaxed);
347        (last > 0).then(|| now_ms().saturating_sub(last))
348    }
349
350    async fn wait_since_release(&mut self, gap: Duration) {
351        if let Some(t) = self.last_release {
352            let since = t.elapsed();
353            if since < gap {
354                tokio::time::sleep(gap - since).await;
355            }
356        }
357    }
358
359    /// Before closing: let the last release sink in.
360    async fn settle_previous(&mut self) {
361        self.wait_since_release(SETTLE).await;
362    }
363
364    /// Before a new touch: keep it apart from the previous one.
365    async fn gap(&mut self) {
366        self.wait_since_release(TOUCH_GAP).await;
367    }
368
369    /// How many touch and button reports this session has written. A gesture
370    /// that fails without this number moving sent nothing to the phone.
371    pub fn reports_sent(&self) -> u64 {
372        self.reports
373    }
374
375    async fn contact(&mut self, x: u16, y: u16) -> Result<()> {
376        crate::device::ax::trace(|| format!("contact {x},{y}"));
377        self.hid
378            .send_touchscreen(TOUCHSCREEN_STATE_CONTACT, x, y, None)
379            .await
380            .map_err(|e| anyhow!("touch contact: {e:?}"))?;
381        self.reports += 1;
382        Ok(())
383    }
384
385    async fn release(&mut self, x: u16, y: u16) -> Result<()> {
386        let r = self
387            .hid
388            .send_touchscreen(TOUCHSCREEN_STATE_RELEASE, x, y, None)
389            .await
390            .map_err(|e| anyhow!("touch release: {e:?}"));
391        crate::device::ax::trace(|| "touch released".into());
392        self.last_release = Some(Instant::now());
393        if r.is_ok() {
394            self.reports += 1;
395        }
396        r
397    }
398
399    /// One finger down at (x, y), held for `hold`, then lifted. Coordinates are
400    /// normalized 0..=65535.
401    pub async fn tap(&mut self, x: u16, y: u16, hold: Duration) -> Result<()> {
402        self.gap().await;
403        crate::device::ax::trace(|| format!("tap down {x},{y}"));
404        self.contact(x, y).await?;
405        tokio::time::sleep(hold).await;
406        self.release(x, y).await
407    }
408
409    pub async fn double_tap(&mut self, x: u16, y: u16) -> Result<()> {
410        self.gap().await;
411        for i in 0..2 {
412            self.contact(x, y).await?;
413            tokio::time::sleep(Duration::from_millis(50)).await;
414            self.release(x, y).await?;
415            if i == 0 {
416                tokio::time::sleep(Duration::from_millis(80)).await;
417            }
418        }
419        Ok(())
420    }
421
422    /// Move one finger from `from` to `to` over `duration`, then lift.
423    ///
424    /// [`Motion::Flick`] moves at constant speed and lifts while moving, so a
425    /// list keeps coasting (a swipe). [`Motion::Place`] slows to a stop before
426    /// lifting, so the content ends up moved by the finger's distance and
427    /// stays there (a scroll step, a drag-and-drop).
428    pub async fn drag(
429        &mut self,
430        from: (u16, u16),
431        to: (u16, u16),
432        duration: Duration,
433        motion: Motion,
434    ) -> Result<()> {
435        self.gap().await;
436        let steps = (duration.as_millis() / 16).clamp(4, 240) as u32;
437        let start = Instant::now();
438        crate::device::ax::trace(|| format!("drag start {from:?} -> {to:?}"));
439        let mut sent_any = false;
440        let mut last = from;
441        let res: Result<()> = async {
442            for i in 0..=steps {
443                let t = i as f64 / steps as f64;
444                let eased = match motion {
445                    Motion::Flick => t,
446                    // Cubic ease-out: fast at first, almost still at the end.
447                    Motion::Place => 1.0 - (1.0 - t).powi(3),
448                };
449                let x = (from.0 as f64 + (to.0 as f64 - from.0 as f64) * eased).round() as u16;
450                let y = (from.1 as f64 + (to.1 as f64 - from.1 as f64) * eased).round() as u16;
451                self.contact(x, y).await?;
452                sent_any = true;
453                last = (x, y);
454                let due = duration.mul_f64(t);
455                let elapsed = start.elapsed();
456                if due > elapsed {
457                    tokio::time::sleep(due - elapsed).await;
458                }
459            }
460            if motion == Motion::Place {
461                // Rest on the end point; the last samples must say "not
462                // moving" or the list flings on release.
463                for _ in 0..4 {
464                    tokio::time::sleep(Duration::from_millis(30)).await;
465                    self.contact(to.0, to.1).await?;
466                }
467            }
468            Ok(())
469        }
470        .await;
471        if sent_any {
472            // Always lift, also after a failed sample.
473            let lifted = self.release(last.0, last.1).await;
474            res?;
475            lifted?;
476        } else {
477            res?;
478        }
479        Ok(())
480    }
481
482    /// Two fingers moving together or turning, then lifted. Every frame counts
483    /// in [`reports_sent`](Self::reports_sent).
484    pub async fn pinch(&mut self, plan: &pinch::Plan) -> Result<()> {
485        self.gap().await;
486        crate::device::ax::trace(|| format!("pinch {} frames, span {:?}", plan.frames.len(), plan.span));
487        let (hid, reports) = (&mut self.hid, &mut self.reports);
488        let played = pinch::play(plan, &mut self.pair_held, async |contacts: &[TouchscreenContact]| -> Result<()> {
489            hid.send_multitouch(contacts, None)
490                .await
491                .map_err(|e| anyhow!("touch pinch: {e:?}"))?;
492            *reports += 1;
493            Ok(())
494        })
495        .await;
496        self.last_release = Some(Instant::now());
497        played
498    }
499
500    /// Lift a pinch that was cut off, so no finger stays down on the phone.
501    /// Bounded: a session given up after an error may have a dead connection.
502    async fn lift_held(&mut self) {
503        let Some(pair) = self.pair_held.take() else {
504            return;
505        };
506        let lifted = pinch::lifted(&pair);
507        let _ = tokio::time::timeout(Duration::from_secs(2), self.hid.send_multitouch(&lifted, None)).await;
508        self.last_release = Some(Instant::now());
509    }
510
511    /// Press and release a hardware button (HID usage page / usage).
512    pub async fn button(&mut self, usage_page: u64, usage: u64, hold: Duration) -> Result<()> {
513        if self.indigo.is_none() {
514            let indigo = IndigoHidClient::connect_rsd(&mut self.adapter, &mut self.rsd)
515                .await
516                .map_err(|e| anyhow!("indigo HID service unavailable: {e:?}"))?;
517            self.indigo = Some(indigo);
518            // Its devices come after those already asked for, so the touch
519            // connection's answer comes once they are made too.
520            let ready = devices_ready(&mut self.hid).await;
521            tokio::time::sleep(WARMUP).await;
522            crate::device::ax::trace(|| format!("input: buttons connected, devices ready at {ready}"));
523        }
524        let Some(indigo) = self.indigo.as_mut() else {
525            unreachable!()
526        };
527        indigo
528            .send_button(usage_page, usage, ButtonState::Down)
529            .await
530            .map_err(|e| anyhow!("button down: {e:?}"))?;
531        self.reports += 1;
532        tokio::time::sleep(hold).await;
533        indigo
534            .send_button(usage_page, usage, ButtonState::Up)
535            .await
536            .map_err(|e| anyhow!("button up: {e:?}"))?;
537        self.last_release = Some(Instant::now());
538        Ok(())
539    }
540
541    /// Register a temporary hardware keyboard with the phone. While it is
542    /// registered iOS hides the on-screen keyboard, so pair every attach with
543    /// [`keyboard_detach`](Self::keyboard_detach).
544    pub async fn keyboard_attach(&mut self) -> Result<()> {
545        if self.board.is_some() {
546            return Ok(());
547        }
548        if self.keyboard.is_none() {
549            let mut keyboard = UniversalHidServiceClient::connect_rsd(&mut self.adapter, &mut self.rsd)
550                .await
551                .map_err(|e| anyhow!("hid service (keyboard): {e:?}"))?;
552            let ready = devices_ready(&mut keyboard).await;
553            tokio::time::sleep(WARMUP).await;
554            crate::device::ax::trace(|| format!("keyboard: connected, devices ready at {ready}"));
555            self.keyboard = Some(keyboard);
556        }
557        let Some(keyboard) = self.keyboard.as_mut() else {
558            unreachable!()
559        };
560        crate::device::ax::trace(|| "keyboard: registering".into());
561        let board = keyboard
562            .create_main_keyboard()
563            .await
564            .map_err(|e| anyhow!("register keyboard: {e}"))?;
565        self.board = Some(board);
566        // The phone needs a moment to attach the new keyboard; keys sent
567        // 250 ms after registering were lost, 800 ms worked (iOS 27.0.1).
568        tokio::time::sleep(KEYBOARD_ATTACH).await;
569        crate::device::ax::trace(|| "keyboard: registered".into());
570        Ok(())
571    }
572
573    /// Press key combinations one after another on the attached keyboard.
574    /// Each is a HID keyboard usage with the modifier usages held around it,
575    /// e.g. `(0x19, [0xE3])` for Command-V.
576    pub async fn keyboard_press(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
577        let usage = |raw: u64| {
578            KeyboardUsage::new(raw as u16).map_err(|e| anyhow!("key usage {raw:#x}: {e}"))
579        };
580        // Validate everything before the first key goes out.
581        for (key, modifiers) in combos {
582            usage(*key)?;
583            for m in modifiers {
584                usage(*m)?;
585            }
586        }
587        let (Some(board), Some(keyboard)) = (self.board.as_mut(), self.keyboard.as_mut()) else {
588            return Err(anyhow!("no keyboard attached"));
589        };
590        for (key, modifiers) in combos {
591            crate::device::ax::trace(|| format!("keyboard: combo {key:#x}"));
592            for m in modifiers {
593                keyboard
594                    .main_keyboard_key_down(board, usage(*m)?)
595                    .await
596                    .map_err(|e| anyhow!("modifier down: {e}"))?;
597                tokio::time::sleep(Duration::from_millis(20)).await;
598            }
599            keyboard
600                .main_keyboard_key_down(board, usage(*key)?)
601                .await
602                .map_err(|e| anyhow!("key down: {e}"))?;
603            tokio::time::sleep(Duration::from_millis(40)).await;
604            // Everything up at once.
605            keyboard
606                .set_main_keyboard_usages(board, std::iter::empty())
607                .await
608                .map_err(|e| anyhow!("key up: {e}"))?;
609            tokio::time::sleep(Duration::from_millis(80)).await;
610        }
611        Ok(())
612    }
613
614    /// Remove the temporary keyboard; this also releases any key still down
615    /// and brings the on-screen keyboard back.
616    pub async fn keyboard_detach(&mut self) -> Result<()> {
617        let Some(mut board) = self.board.take() else {
618            return Ok(());
619        };
620        let Some(keyboard) = self.keyboard.as_mut() else {
621            return Ok(());
622        };
623        crate::device::ax::trace(|| "keyboard: removing".into());
624        let removed = keyboard.remove_main_keyboard(&mut board).await;
625        crate::device::ax::trace(|| "keyboard: removed".into());
626        removed.map_err(|e| anyhow!("remove keyboard: {e}"))
627    }
628
629    /// Attach, press, detach.
630    pub async fn keys(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
631        self.keyboard_attach().await?;
632        let pressed = self.keyboard_press(combos).await;
633        // Remove the keyboard whatever happened.
634        let detached = self.keyboard_detach().await;
635        pressed?;
636        detached
637    }
638
639    /// One key with optional modifiers.
640    pub async fn key(&mut self, usage: u64, modifiers: &[u64]) -> Result<()> {
641        self.keys(&[(usage, modifiers.to_vec())]).await
642    }
643
644    pub async fn close(mut self) {
645        self.lift_held().await;
646        let _ = self.keyboard_detach().await;
647        self.settle_previous().await;
648        let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
649    }
650
651    /// Stop the stream of a session given up after an error, without the
652    /// orderly wait. A stream nobody stops keeps the phone encoding video.
653    pub async fn abandon(mut self) {
654        self.lift_held().await;
655        let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
656    }
657}