1use std::{
12 sync::{
13 Arc,
14 atomic::{AtomicU64, Ordering},
15 },
16 time::{Duration, Instant},
17};
18
19use anyhow::{Result, anyhow};
20use idevice::{
21 ReadWrite, RsdService,
22 core_device::{
23 CallInfoBlob, DisplayServiceClient, build_screen_audio_offer, build_screen_video_offer,
24 build_start_audio_parameters, build_start_video_parameters,
25 hid::{
26 ButtonState, IndigoHidClient, KeyboardUsage, MainKeyboardService,
27 TOUCHSCREEN_STATE_CONTACT, TOUCHSCREEN_STATE_RELEASE, TouchscreenContact,
28 UniversalHidServiceClient,
29 },
30 },
31 rsd::RsdHandshake,
32};
33
34use crate::{
35 device::pinch,
36 tcp::handle::{AdapterHandle, UdpSocketHandle},
37};
38
39const CLIENT_SUPPORTED_FEATURES: u64 = 140;
41
42const DEVICES_READY: Duration = Duration::from_secs(5);
44const WARMUP: Duration = Duration::from_millis(300);
53const SETTLE: Duration = Duration::from_millis(300);
58const TOUCH_GAP: Duration = Duration::from_millis(150);
61
62const FIRST_PACKET: Duration = Duration::from_secs(3);
65
66const KEYBOARD_ATTACH: Duration = Duration::from_millis(700);
68
69fn warmup() -> Duration {
71 std::env::var("IPH_INPUT_WARMUP_MS")
72 .ok()
73 .and_then(|v| v.parse().ok())
74 .map(Duration::from_millis)
75 .unwrap_or(WARMUP)
76}
77
78async fn devices_ready<R: ReadWrite>(client: &mut UniversalHidServiceClient<R>) -> String {
81 let began = now_ms();
82 match tokio::time::timeout(DEVICES_READY, client.list_connected_services()).await {
83 Ok(Ok(_)) => format!("{} ms", now_ms().saturating_sub(began)),
84 Ok(Err(e)) => format!("error {e:?}"),
85 Err(_) => "timed out".to_string(),
86 }
87}
88
89fn now_ms() -> u64 {
90 std::time::SystemTime::now()
91 .duration_since(std::time::UNIX_EPOCH)
92 .map(|d| d.as_millis() as u64)
93 .unwrap_or(0)
94}
95
96struct Drain {
97 last_packet_ms: Arc<AtomicU64>,
98 first_packet_ms: Vec<Arc<AtomicU64>>,
100 tasks: Vec<tokio::task::JoinHandle<()>>,
101}
102
103impl Drain {
104 fn start(sockets: Vec<UdpSocketHandle>) -> Self {
105 let last_packet_ms = Arc::new(AtomicU64::new(0));
106 let first_packet_ms: Vec<Arc<AtomicU64>> =
107 sockets.iter().map(|_| Arc::new(AtomicU64::new(0))).collect();
108 let counts: Vec<Arc<(AtomicU64, AtomicU64)>> =
112 sockets.iter().map(|_| Arc::new((AtomicU64::new(0), AtomicU64::new(0)))).collect();
113 let mut tasks: Vec<tokio::task::JoinHandle<()>> = sockets
114 .into_iter()
115 .zip(first_packet_ms.iter().cloned())
116 .zip(counts.iter().cloned())
117 .map(|((s, first), count)| {
118 let last = last_packet_ms.clone();
119 tokio::spawn(async move {
120 while let Ok(packet) = s.recv().await {
122 let now = now_ms();
123 last.store(now, Ordering::Relaxed);
124 let _ = first.compare_exchange(0, now, Ordering::Relaxed, Ordering::Relaxed);
125 count.0.fetch_add(packet.data.len() as u64, Ordering::Relaxed);
126 count.1.fetch_add(1, Ordering::Relaxed);
127 }
128 })
129 })
130 .collect();
131 if std::env::var("IPH_TRACE").is_ok_and(|v| v == "1") {
132 tasks.push(tokio::spawn(async move {
133 let mut before: Vec<(u64, u64)> = counts.iter().map(|_| (0, 0)).collect();
134 loop {
135 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
136 let now: Vec<(u64, u64)> = counts
137 .iter()
138 .map(|c| (c.0.load(Ordering::Relaxed), c.1.load(Ordering::Relaxed)))
139 .collect();
140 let rates: Vec<String> = now
141 .iter()
142 .zip(&before)
143 .map(|(n, b)| format!("{} B/{} pkts", n.0 - b.0, n.1 - b.1))
144 .collect();
145 crate::device::ax::trace(|| format!("media in the last second: {}", rates.join(", ")));
146 before = now;
147 }
148 }));
149 }
150 Self {
151 last_packet_ms,
152 first_packet_ms,
153 tasks,
154 }
155 }
156
157 fn flowing(&self) -> bool {
159 self.first_packet_ms.iter().all(|f| f.load(Ordering::Relaxed) > 0)
160 }
161}
162
163impl Drop for Drain {
164 fn drop(&mut self) {
165 for t in &self.tasks {
166 t.abort();
167 }
168 }
169}
170
171#[derive(Debug, Clone, Copy, PartialEq, Eq)]
172pub enum Motion {
173 Flick,
174 Place,
175}
176
177pub struct InputSession {
178 display: DisplayServiceClient<Box<dyn ReadWrite>>,
179 hid: UniversalHidServiceClient<Box<dyn ReadWrite>>,
180 keyboard: Option<UniversalHidServiceClient<Box<dyn ReadWrite>>>,
183 board: Option<MainKeyboardService>,
185 indigo: Option<IndigoHidClient<Box<dyn ReadWrite>>>,
187 adapter: AdapterHandle,
189 rsd: RsdHandshake,
190 drain: Drain,
191 pub started: Instant,
192 last_release: Option<Instant>,
193 reports: u64,
195 pair_held: Option<[TouchscreenContact; 2]>,
197}
198
199pub fn normalize(v: f64, extent: f64) -> u16 {
201 if extent <= 0.0 {
202 return 0;
203 }
204 ((v / extent) * 65535.0).round().clamp(0.0, 65535.0) as u16
205}
206
207impl InputSession {
208 pub async fn open(adapter: &mut AdapterHandle, rsd: &mut RsdHandshake, video: bool) -> Result<Self> {
215 let began = Instant::now();
216 let began_ms = now_ms();
217 let mut display = DisplayServiceClient::connect_rsd(adapter, rsd)
218 .await
219 .map_err(|e| anyhow!("display service: {e:?}"))?;
220
221 let audio_udp = adapter.bind_udp(0).await?;
222 let video_udp = adapter.bind_udp(0).await?;
223 let receiver_ip = adapter.host_ip().to_string();
224 let sender_ip = adapter.peer_ip().to_string();
225
226 let call_info = CallInfoBlob {
227 call_id: 0,
228 client_version: 1,
229 device_type: "Mac17,7".into(),
230 framework_version: "2205.3.1".into(),
231 os_version: "25F71".into(),
232 device_name: None,
233 audio_device_uid: None,
234 };
235 let session_id = uuid::Uuid::new_v4();
236
237 let audio_call = uuid::Uuid::new_v4().to_string().to_uppercase();
238 let audio_offer = build_screen_audio_offer(&audio_call, &call_info)
239 .map_err(|e| anyhow!("audio offer: {e:?}"))?;
240 let audio = display
241 .start_media_stream(build_start_audio_parameters(
242 &receiver_ip,
243 audio_udp.local_port(),
244 &sender_ip,
245 50000,
246 audio_offer,
247 CLIENT_SUPPORTED_FEATURES,
248 session_id,
249 ))
250 .await;
251 let mut streams = Vec::new();
252 let mut video = video;
253 match audio {
254 Ok(_) => streams.push(audio_udp),
255 Err(e) if format!("{e:?}").contains("Integer(9022)") => video = true,
259 Err(e) => return Err(anyhow!("start audio stream: {e:?}")),
260 }
261 if video {
262 let video_call = uuid::Uuid::new_v4().to_string().to_uppercase();
263 let ssrc = uuid::Uuid::new_v4().as_u128() as u32;
264 let video_offer = build_screen_video_offer(&video_call, &call_info, ssrc)
265 .map_err(|e| anyhow!("video offer: {e:?}"))?;
266 display
267 .start_media_stream(build_start_video_parameters(
268 &receiver_ip,
269 video_udp.local_port(),
270 &sender_ip,
271 50001,
272 video_offer,
273 CLIENT_SUPPORTED_FEATURES,
274 1,
275 session_id,
276 ))
277 .await
278 .map_err(|e| anyhow!("start video stream: {e:?}"))?;
279 streams.push(video_udp);
280 }
281
282 let drain = Drain::start(streams);
283
284 let mut hid = UniversalHidServiceClient::connect_rsd(adapter, rsd)
289 .await
290 .map_err(|e| anyhow!("hid service: {e:?}"))?;
291 let hid_ready = devices_ready(&mut hid).await;
292 let prime = std::env::var("IPH_INPUT_PRIME").map(|v| v != "0").unwrap_or(true);
298 if prime {
299 let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
300 }
301 tokio::time::sleep(warmup()).await;
302 let waiting = Instant::now();
305 while !drain.flowing() && waiting.elapsed() < FIRST_PACKET {
306 tokio::time::sleep(Duration::from_millis(50)).await;
307 }
308 crate::device::ax::trace(|| {
309 let firsts: Vec<String> = drain
310 .first_packet_ms
311 .iter()
312 .map(|f| match f.load(Ordering::Relaxed) {
313 0 => "none".to_string(),
314 at => format!("{} ms", at.saturating_sub(began_ms)),
315 })
316 .collect();
317 format!(
318 "input: session open after {} ms, devices ready at {hid_ready}, first audio/video packet at {}",
319 began.elapsed().as_millis(),
320 firsts.join(" / ")
321 )
322 });
323 if prime {
324 let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
325 tokio::time::sleep(Duration::from_millis(60)).await;
326 }
327
328 Ok(Self {
329 display,
330 hid,
331 keyboard: None,
332 board: None,
333 indigo: None,
334 adapter: adapter.clone(),
335 rsd: rsd.clone(),
336 drain,
337 started: Instant::now(),
338 last_release: None,
339 reports: 0,
340 pair_held: None,
341 })
342 }
343
344 pub fn media_idle_ms(&self) -> Option<u64> {
346 let last = self.drain.last_packet_ms.load(Ordering::Relaxed);
347 (last > 0).then(|| now_ms().saturating_sub(last))
348 }
349
350 async fn wait_since_release(&mut self, gap: Duration) {
351 if let Some(t) = self.last_release {
352 let since = t.elapsed();
353 if since < gap {
354 tokio::time::sleep(gap - since).await;
355 }
356 }
357 }
358
359 async fn settle_previous(&mut self) {
361 self.wait_since_release(SETTLE).await;
362 }
363
364 async fn gap(&mut self) {
366 self.wait_since_release(TOUCH_GAP).await;
367 }
368
369 pub fn reports_sent(&self) -> u64 {
372 self.reports
373 }
374
375 async fn contact(&mut self, x: u16, y: u16) -> Result<()> {
376 crate::device::ax::trace(|| format!("contact {x},{y}"));
377 self.hid
378 .send_touchscreen(TOUCHSCREEN_STATE_CONTACT, x, y, None)
379 .await
380 .map_err(|e| anyhow!("touch contact: {e:?}"))?;
381 self.reports += 1;
382 Ok(())
383 }
384
385 async fn release(&mut self, x: u16, y: u16) -> Result<()> {
386 let r = self
387 .hid
388 .send_touchscreen(TOUCHSCREEN_STATE_RELEASE, x, y, None)
389 .await
390 .map_err(|e| anyhow!("touch release: {e:?}"));
391 crate::device::ax::trace(|| "touch released".into());
392 self.last_release = Some(Instant::now());
393 if r.is_ok() {
394 self.reports += 1;
395 }
396 r
397 }
398
399 pub async fn tap(&mut self, x: u16, y: u16, hold: Duration) -> Result<()> {
402 self.gap().await;
403 crate::device::ax::trace(|| format!("tap down {x},{y}"));
404 self.contact(x, y).await?;
405 tokio::time::sleep(hold).await;
406 self.release(x, y).await
407 }
408
409 pub async fn double_tap(&mut self, x: u16, y: u16) -> Result<()> {
410 self.gap().await;
411 for i in 0..2 {
412 self.contact(x, y).await?;
413 tokio::time::sleep(Duration::from_millis(50)).await;
414 self.release(x, y).await?;
415 if i == 0 {
416 tokio::time::sleep(Duration::from_millis(80)).await;
417 }
418 }
419 Ok(())
420 }
421
422 pub async fn drag(
429 &mut self,
430 from: (u16, u16),
431 to: (u16, u16),
432 duration: Duration,
433 motion: Motion,
434 ) -> Result<()> {
435 self.gap().await;
436 let steps = (duration.as_millis() / 16).clamp(4, 240) as u32;
437 let start = Instant::now();
438 crate::device::ax::trace(|| format!("drag start {from:?} -> {to:?}"));
439 let mut sent_any = false;
440 let mut last = from;
441 let res: Result<()> = async {
442 for i in 0..=steps {
443 let t = i as f64 / steps as f64;
444 let eased = match motion {
445 Motion::Flick => t,
446 Motion::Place => 1.0 - (1.0 - t).powi(3),
448 };
449 let x = (from.0 as f64 + (to.0 as f64 - from.0 as f64) * eased).round() as u16;
450 let y = (from.1 as f64 + (to.1 as f64 - from.1 as f64) * eased).round() as u16;
451 self.contact(x, y).await?;
452 sent_any = true;
453 last = (x, y);
454 let due = duration.mul_f64(t);
455 let elapsed = start.elapsed();
456 if due > elapsed {
457 tokio::time::sleep(due - elapsed).await;
458 }
459 }
460 if motion == Motion::Place {
461 for _ in 0..4 {
464 tokio::time::sleep(Duration::from_millis(30)).await;
465 self.contact(to.0, to.1).await?;
466 }
467 }
468 Ok(())
469 }
470 .await;
471 if sent_any {
472 let lifted = self.release(last.0, last.1).await;
474 res?;
475 lifted?;
476 } else {
477 res?;
478 }
479 Ok(())
480 }
481
482 pub async fn pinch(&mut self, plan: &pinch::Plan) -> Result<()> {
485 self.gap().await;
486 crate::device::ax::trace(|| format!("pinch {} frames, span {:?}", plan.frames.len(), plan.span));
487 let (hid, reports) = (&mut self.hid, &mut self.reports);
488 let played = pinch::play(plan, &mut self.pair_held, async |contacts: &[TouchscreenContact]| -> Result<()> {
489 hid.send_multitouch(contacts, None)
490 .await
491 .map_err(|e| anyhow!("touch pinch: {e:?}"))?;
492 *reports += 1;
493 Ok(())
494 })
495 .await;
496 self.last_release = Some(Instant::now());
497 played
498 }
499
500 async fn lift_held(&mut self) {
503 let Some(pair) = self.pair_held.take() else {
504 return;
505 };
506 let lifted = pinch::lifted(&pair);
507 let _ = tokio::time::timeout(Duration::from_secs(2), self.hid.send_multitouch(&lifted, None)).await;
508 self.last_release = Some(Instant::now());
509 }
510
511 pub async fn button(&mut self, usage_page: u64, usage: u64, hold: Duration) -> Result<()> {
513 if self.indigo.is_none() {
514 let indigo = IndigoHidClient::connect_rsd(&mut self.adapter, &mut self.rsd)
515 .await
516 .map_err(|e| anyhow!("indigo HID service unavailable: {e:?}"))?;
517 self.indigo = Some(indigo);
518 let ready = devices_ready(&mut self.hid).await;
521 tokio::time::sleep(WARMUP).await;
522 crate::device::ax::trace(|| format!("input: buttons connected, devices ready at {ready}"));
523 }
524 let Some(indigo) = self.indigo.as_mut() else {
525 unreachable!()
526 };
527 indigo
528 .send_button(usage_page, usage, ButtonState::Down)
529 .await
530 .map_err(|e| anyhow!("button down: {e:?}"))?;
531 self.reports += 1;
532 tokio::time::sleep(hold).await;
533 indigo
534 .send_button(usage_page, usage, ButtonState::Up)
535 .await
536 .map_err(|e| anyhow!("button up: {e:?}"))?;
537 self.last_release = Some(Instant::now());
538 Ok(())
539 }
540
541 pub async fn keyboard_attach(&mut self) -> Result<()> {
545 if self.board.is_some() {
546 return Ok(());
547 }
548 if self.keyboard.is_none() {
549 let mut keyboard = UniversalHidServiceClient::connect_rsd(&mut self.adapter, &mut self.rsd)
550 .await
551 .map_err(|e| anyhow!("hid service (keyboard): {e:?}"))?;
552 let ready = devices_ready(&mut keyboard).await;
553 tokio::time::sleep(WARMUP).await;
554 crate::device::ax::trace(|| format!("keyboard: connected, devices ready at {ready}"));
555 self.keyboard = Some(keyboard);
556 }
557 let Some(keyboard) = self.keyboard.as_mut() else {
558 unreachable!()
559 };
560 crate::device::ax::trace(|| "keyboard: registering".into());
561 let board = keyboard
562 .create_main_keyboard()
563 .await
564 .map_err(|e| anyhow!("register keyboard: {e}"))?;
565 self.board = Some(board);
566 tokio::time::sleep(KEYBOARD_ATTACH).await;
569 crate::device::ax::trace(|| "keyboard: registered".into());
570 Ok(())
571 }
572
573 pub async fn keyboard_press(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
577 let usage = |raw: u64| {
578 KeyboardUsage::new(raw as u16).map_err(|e| anyhow!("key usage {raw:#x}: {e}"))
579 };
580 for (key, modifiers) in combos {
582 usage(*key)?;
583 for m in modifiers {
584 usage(*m)?;
585 }
586 }
587 let (Some(board), Some(keyboard)) = (self.board.as_mut(), self.keyboard.as_mut()) else {
588 return Err(anyhow!("no keyboard attached"));
589 };
590 for (key, modifiers) in combos {
591 crate::device::ax::trace(|| format!("keyboard: combo {key:#x}"));
592 for m in modifiers {
593 keyboard
594 .main_keyboard_key_down(board, usage(*m)?)
595 .await
596 .map_err(|e| anyhow!("modifier down: {e}"))?;
597 tokio::time::sleep(Duration::from_millis(20)).await;
598 }
599 keyboard
600 .main_keyboard_key_down(board, usage(*key)?)
601 .await
602 .map_err(|e| anyhow!("key down: {e}"))?;
603 tokio::time::sleep(Duration::from_millis(40)).await;
604 keyboard
606 .set_main_keyboard_usages(board, std::iter::empty())
607 .await
608 .map_err(|e| anyhow!("key up: {e}"))?;
609 tokio::time::sleep(Duration::from_millis(80)).await;
610 }
611 Ok(())
612 }
613
614 pub async fn keyboard_detach(&mut self) -> Result<()> {
617 let Some(mut board) = self.board.take() else {
618 return Ok(());
619 };
620 let Some(keyboard) = self.keyboard.as_mut() else {
621 return Ok(());
622 };
623 crate::device::ax::trace(|| "keyboard: removing".into());
624 let removed = keyboard.remove_main_keyboard(&mut board).await;
625 crate::device::ax::trace(|| "keyboard: removed".into());
626 removed.map_err(|e| anyhow!("remove keyboard: {e}"))
627 }
628
629 pub async fn keys(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
631 self.keyboard_attach().await?;
632 let pressed = self.keyboard_press(combos).await;
633 let detached = self.keyboard_detach().await;
635 pressed?;
636 detached
637 }
638
639 pub async fn key(&mut self, usage: u64, modifiers: &[u64]) -> Result<()> {
641 self.keys(&[(usage, modifiers.to_vec())]).await
642 }
643
644 pub async fn close(mut self) {
645 self.lift_held().await;
646 let _ = self.keyboard_detach().await;
647 self.settle_previous().await;
648 let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
649 }
650
651 pub async fn abandon(mut self) {
654 self.lift_held().await;
655 let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
656 }
657}