1use std::{
12 sync::{
13 Arc,
14 atomic::{AtomicU64, Ordering},
15 },
16 time::{Duration, Instant},
17};
18
19use anyhow::{Result, anyhow};
20use idevice::{
21 ReadWrite, RsdService,
22 core_device::{
23 CallInfoBlob, DisplayServiceClient, build_screen_audio_offer, build_screen_video_offer,
24 build_start_audio_parameters, build_start_video_parameters,
25 hid::{
26 ButtonState, IndigoHidClient, KeyboardUsage, MainKeyboardService,
27 TOUCHSCREEN_STATE_CONTACT, TOUCHSCREEN_STATE_RELEASE, UniversalHidServiceClient,
28 },
29 },
30 rsd::RsdHandshake,
31};
32
33use crate::tcp::handle::{AdapterHandle, UdpSocketHandle};
34
35const CLIENT_SUPPORTED_FEATURES: u64 = 140;
37
38const DEVICES_READY: Duration = Duration::from_secs(5);
40const WARMUP: Duration = Duration::from_millis(300);
49const SETTLE: Duration = Duration::from_millis(300);
54const TOUCH_GAP: Duration = Duration::from_millis(150);
57
58const FIRST_PACKET: Duration = Duration::from_secs(3);
61
62const KEYBOARD_ATTACH: Duration = Duration::from_millis(700);
64
65fn warmup() -> Duration {
67 std::env::var("IPH_INPUT_WARMUP_MS")
68 .ok()
69 .and_then(|v| v.parse().ok())
70 .map(Duration::from_millis)
71 .unwrap_or(WARMUP)
72}
73
74async fn devices_ready<R: ReadWrite>(client: &mut UniversalHidServiceClient<R>) -> String {
77 let began = now_ms();
78 match tokio::time::timeout(DEVICES_READY, client.list_connected_services()).await {
79 Ok(Ok(_)) => format!("{} ms", now_ms().saturating_sub(began)),
80 Ok(Err(e)) => format!("error {e:?}"),
81 Err(_) => "timed out".to_string(),
82 }
83}
84
85fn now_ms() -> u64 {
86 std::time::SystemTime::now()
87 .duration_since(std::time::UNIX_EPOCH)
88 .map(|d| d.as_millis() as u64)
89 .unwrap_or(0)
90}
91
92struct Drain {
93 last_packet_ms: Arc<AtomicU64>,
94 first_packet_ms: Vec<Arc<AtomicU64>>,
96 tasks: Vec<tokio::task::JoinHandle<()>>,
97}
98
99impl Drain {
100 fn start(sockets: Vec<UdpSocketHandle>) -> Self {
101 let last_packet_ms = Arc::new(AtomicU64::new(0));
102 let first_packet_ms: Vec<Arc<AtomicU64>> =
103 sockets.iter().map(|_| Arc::new(AtomicU64::new(0))).collect();
104 let counts: Vec<Arc<(AtomicU64, AtomicU64)>> =
108 sockets.iter().map(|_| Arc::new((AtomicU64::new(0), AtomicU64::new(0)))).collect();
109 let mut tasks: Vec<tokio::task::JoinHandle<()>> = sockets
110 .into_iter()
111 .zip(first_packet_ms.iter().cloned())
112 .zip(counts.iter().cloned())
113 .map(|((s, first), count)| {
114 let last = last_packet_ms.clone();
115 tokio::spawn(async move {
116 while let Ok(packet) = s.recv().await {
118 let now = now_ms();
119 last.store(now, Ordering::Relaxed);
120 let _ = first.compare_exchange(0, now, Ordering::Relaxed, Ordering::Relaxed);
121 count.0.fetch_add(packet.data.len() as u64, Ordering::Relaxed);
122 count.1.fetch_add(1, Ordering::Relaxed);
123 }
124 })
125 })
126 .collect();
127 if std::env::var("IPH_TRACE").is_ok_and(|v| v == "1") {
128 tasks.push(tokio::spawn(async move {
129 let mut before: Vec<(u64, u64)> = counts.iter().map(|_| (0, 0)).collect();
130 loop {
131 tokio::time::sleep(std::time::Duration::from_secs(1)).await;
132 let now: Vec<(u64, u64)> = counts
133 .iter()
134 .map(|c| (c.0.load(Ordering::Relaxed), c.1.load(Ordering::Relaxed)))
135 .collect();
136 let rates: Vec<String> = now
137 .iter()
138 .zip(&before)
139 .map(|(n, b)| format!("{} B/{} pkts", n.0 - b.0, n.1 - b.1))
140 .collect();
141 crate::device::ax::trace(|| format!("media in the last second: {}", rates.join(", ")));
142 before = now;
143 }
144 }));
145 }
146 Self {
147 last_packet_ms,
148 first_packet_ms,
149 tasks,
150 }
151 }
152
153 fn flowing(&self) -> bool {
155 self.first_packet_ms.iter().all(|f| f.load(Ordering::Relaxed) > 0)
156 }
157}
158
159impl Drop for Drain {
160 fn drop(&mut self) {
161 for t in &self.tasks {
162 t.abort();
163 }
164 }
165}
166
167#[derive(Debug, Clone, Copy, PartialEq, Eq)]
168pub enum Motion {
169 Flick,
170 Place,
171}
172
173pub struct InputSession {
174 display: DisplayServiceClient<Box<dyn ReadWrite>>,
175 hid: UniversalHidServiceClient<Box<dyn ReadWrite>>,
176 keyboard: Option<UniversalHidServiceClient<Box<dyn ReadWrite>>>,
179 board: Option<MainKeyboardService>,
181 indigo: Option<IndigoHidClient<Box<dyn ReadWrite>>>,
183 adapter: AdapterHandle,
185 rsd: RsdHandshake,
186 drain: Drain,
187 pub started: Instant,
188 last_release: Option<Instant>,
189 reports: u64,
191}
192
193pub fn normalize(v: f64, extent: f64) -> u16 {
195 if extent <= 0.0 {
196 return 0;
197 }
198 ((v / extent) * 65535.0).round().clamp(0.0, 65535.0) as u16
199}
200
201impl InputSession {
202 pub async fn open(adapter: &mut AdapterHandle, rsd: &mut RsdHandshake, video: bool) -> Result<Self> {
209 let began = Instant::now();
210 let began_ms = now_ms();
211 let mut display = DisplayServiceClient::connect_rsd(adapter, rsd)
212 .await
213 .map_err(|e| anyhow!("display service: {e:?}"))?;
214
215 let audio_udp = adapter.bind_udp(0).await?;
216 let video_udp = adapter.bind_udp(0).await?;
217 let receiver_ip = adapter.host_ip().to_string();
218 let sender_ip = adapter.peer_ip().to_string();
219
220 let call_info = CallInfoBlob {
221 call_id: 0,
222 client_version: 1,
223 device_type: "Mac17,7".into(),
224 framework_version: "2205.3.1".into(),
225 os_version: "25F71".into(),
226 device_name: None,
227 audio_device_uid: None,
228 };
229 let session_id = uuid::Uuid::new_v4();
230
231 let audio_call = uuid::Uuid::new_v4().to_string().to_uppercase();
232 let audio_offer = build_screen_audio_offer(&audio_call, &call_info)
233 .map_err(|e| anyhow!("audio offer: {e:?}"))?;
234 let audio = display
235 .start_media_stream(build_start_audio_parameters(
236 &receiver_ip,
237 audio_udp.local_port(),
238 &sender_ip,
239 50000,
240 audio_offer,
241 CLIENT_SUPPORTED_FEATURES,
242 session_id,
243 ))
244 .await;
245 let mut streams = Vec::new();
246 let mut video = video;
247 match audio {
248 Ok(_) => streams.push(audio_udp),
249 Err(e) if format!("{e:?}").contains("Integer(9022)") => video = true,
253 Err(e) => return Err(anyhow!("start audio stream: {e:?}")),
254 }
255 if video {
256 let video_call = uuid::Uuid::new_v4().to_string().to_uppercase();
257 let ssrc = uuid::Uuid::new_v4().as_u128() as u32;
258 let video_offer = build_screen_video_offer(&video_call, &call_info, ssrc)
259 .map_err(|e| anyhow!("video offer: {e:?}"))?;
260 display
261 .start_media_stream(build_start_video_parameters(
262 &receiver_ip,
263 video_udp.local_port(),
264 &sender_ip,
265 50001,
266 video_offer,
267 CLIENT_SUPPORTED_FEATURES,
268 1,
269 session_id,
270 ))
271 .await
272 .map_err(|e| anyhow!("start video stream: {e:?}"))?;
273 streams.push(video_udp);
274 }
275
276 let drain = Drain::start(streams);
277
278 let mut hid = UniversalHidServiceClient::connect_rsd(adapter, rsd)
283 .await
284 .map_err(|e| anyhow!("hid service: {e:?}"))?;
285 let hid_ready = devices_ready(&mut hid).await;
286 let prime = std::env::var("IPH_INPUT_PRIME").map(|v| v != "0").unwrap_or(true);
292 if prime {
293 let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
294 }
295 tokio::time::sleep(warmup()).await;
296 let waiting = Instant::now();
299 while !drain.flowing() && waiting.elapsed() < FIRST_PACKET {
300 tokio::time::sleep(Duration::from_millis(50)).await;
301 }
302 crate::device::ax::trace(|| {
303 let firsts: Vec<String> = drain
304 .first_packet_ms
305 .iter()
306 .map(|f| match f.load(Ordering::Relaxed) {
307 0 => "none".to_string(),
308 at => format!("{} ms", at.saturating_sub(began_ms)),
309 })
310 .collect();
311 format!(
312 "input: session open after {} ms, devices ready at {hid_ready}, first audio/video packet at {}",
313 began.elapsed().as_millis(),
314 firsts.join(" / ")
315 )
316 });
317 if prime {
318 let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
319 tokio::time::sleep(Duration::from_millis(60)).await;
320 }
321
322 Ok(Self {
323 display,
324 hid,
325 keyboard: None,
326 board: None,
327 indigo: None,
328 adapter: adapter.clone(),
329 rsd: rsd.clone(),
330 drain,
331 started: Instant::now(),
332 last_release: None,
333 reports: 0,
334 })
335 }
336
337 pub fn media_idle_ms(&self) -> Option<u64> {
339 let last = self.drain.last_packet_ms.load(Ordering::Relaxed);
340 (last > 0).then(|| now_ms().saturating_sub(last))
341 }
342
343 async fn wait_since_release(&mut self, gap: Duration) {
344 if let Some(t) = self.last_release {
345 let since = t.elapsed();
346 if since < gap {
347 tokio::time::sleep(gap - since).await;
348 }
349 }
350 }
351
352 async fn settle_previous(&mut self) {
354 self.wait_since_release(SETTLE).await;
355 }
356
357 async fn gap(&mut self) {
359 self.wait_since_release(TOUCH_GAP).await;
360 }
361
362 pub fn reports_sent(&self) -> u64 {
365 self.reports
366 }
367
368 async fn contact(&mut self, x: u16, y: u16) -> Result<()> {
369 crate::device::ax::trace(|| format!("contact {x},{y}"));
370 self.hid
371 .send_touchscreen(TOUCHSCREEN_STATE_CONTACT, x, y, None)
372 .await
373 .map_err(|e| anyhow!("touch contact: {e:?}"))?;
374 self.reports += 1;
375 Ok(())
376 }
377
378 async fn release(&mut self, x: u16, y: u16) -> Result<()> {
379 let r = self
380 .hid
381 .send_touchscreen(TOUCHSCREEN_STATE_RELEASE, x, y, None)
382 .await
383 .map_err(|e| anyhow!("touch release: {e:?}"));
384 crate::device::ax::trace(|| "touch released".into());
385 self.last_release = Some(Instant::now());
386 if r.is_ok() {
387 self.reports += 1;
388 }
389 r
390 }
391
392 pub async fn tap(&mut self, x: u16, y: u16, hold: Duration) -> Result<()> {
395 self.gap().await;
396 crate::device::ax::trace(|| format!("tap down {x},{y}"));
397 self.contact(x, y).await?;
398 tokio::time::sleep(hold).await;
399 self.release(x, y).await
400 }
401
402 pub async fn double_tap(&mut self, x: u16, y: u16) -> Result<()> {
403 self.gap().await;
404 for i in 0..2 {
405 self.contact(x, y).await?;
406 tokio::time::sleep(Duration::from_millis(50)).await;
407 self.release(x, y).await?;
408 if i == 0 {
409 tokio::time::sleep(Duration::from_millis(80)).await;
410 }
411 }
412 Ok(())
413 }
414
415 pub async fn drag(
422 &mut self,
423 from: (u16, u16),
424 to: (u16, u16),
425 duration: Duration,
426 motion: Motion,
427 ) -> Result<()> {
428 self.gap().await;
429 let steps = (duration.as_millis() / 16).clamp(4, 240) as u32;
430 let start = Instant::now();
431 crate::device::ax::trace(|| format!("drag start {from:?} -> {to:?}"));
432 let mut sent_any = false;
433 let mut last = from;
434 let res: Result<()> = async {
435 for i in 0..=steps {
436 let t = i as f64 / steps as f64;
437 let eased = match motion {
438 Motion::Flick => t,
439 Motion::Place => 1.0 - (1.0 - t).powi(3),
441 };
442 let x = (from.0 as f64 + (to.0 as f64 - from.0 as f64) * eased).round() as u16;
443 let y = (from.1 as f64 + (to.1 as f64 - from.1 as f64) * eased).round() as u16;
444 self.contact(x, y).await?;
445 sent_any = true;
446 last = (x, y);
447 let due = duration.mul_f64(t);
448 let elapsed = start.elapsed();
449 if due > elapsed {
450 tokio::time::sleep(due - elapsed).await;
451 }
452 }
453 if motion == Motion::Place {
454 for _ in 0..4 {
457 tokio::time::sleep(Duration::from_millis(30)).await;
458 self.contact(to.0, to.1).await?;
459 }
460 }
461 Ok(())
462 }
463 .await;
464 if sent_any {
465 let lifted = self.release(last.0, last.1).await;
467 res?;
468 lifted?;
469 } else {
470 res?;
471 }
472 Ok(())
473 }
474
475 pub async fn button(&mut self, usage_page: u64, usage: u64, hold: Duration) -> Result<()> {
477 if self.indigo.is_none() {
478 let indigo = IndigoHidClient::connect_rsd(&mut self.adapter, &mut self.rsd)
479 .await
480 .map_err(|e| anyhow!("indigo HID service unavailable: {e:?}"))?;
481 self.indigo = Some(indigo);
482 let ready = devices_ready(&mut self.hid).await;
485 tokio::time::sleep(WARMUP).await;
486 crate::device::ax::trace(|| format!("input: buttons connected, devices ready at {ready}"));
487 }
488 let Some(indigo) = self.indigo.as_mut() else {
489 unreachable!()
490 };
491 indigo
492 .send_button(usage_page, usage, ButtonState::Down)
493 .await
494 .map_err(|e| anyhow!("button down: {e:?}"))?;
495 self.reports += 1;
496 tokio::time::sleep(hold).await;
497 indigo
498 .send_button(usage_page, usage, ButtonState::Up)
499 .await
500 .map_err(|e| anyhow!("button up: {e:?}"))?;
501 self.last_release = Some(Instant::now());
502 Ok(())
503 }
504
505 pub async fn keyboard_attach(&mut self) -> Result<()> {
509 if self.board.is_some() {
510 return Ok(());
511 }
512 if self.keyboard.is_none() {
513 let mut keyboard = UniversalHidServiceClient::connect_rsd(&mut self.adapter, &mut self.rsd)
514 .await
515 .map_err(|e| anyhow!("hid service (keyboard): {e:?}"))?;
516 let ready = devices_ready(&mut keyboard).await;
517 tokio::time::sleep(WARMUP).await;
518 crate::device::ax::trace(|| format!("keyboard: connected, devices ready at {ready}"));
519 self.keyboard = Some(keyboard);
520 }
521 let Some(keyboard) = self.keyboard.as_mut() else {
522 unreachable!()
523 };
524 crate::device::ax::trace(|| "keyboard: registering".into());
525 let board = keyboard
526 .create_main_keyboard()
527 .await
528 .map_err(|e| anyhow!("register keyboard: {e}"))?;
529 self.board = Some(board);
530 tokio::time::sleep(KEYBOARD_ATTACH).await;
533 crate::device::ax::trace(|| "keyboard: registered".into());
534 Ok(())
535 }
536
537 pub async fn keyboard_press(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
541 let usage = |raw: u64| {
542 KeyboardUsage::new(raw as u16).map_err(|e| anyhow!("key usage {raw:#x}: {e}"))
543 };
544 for (key, modifiers) in combos {
546 usage(*key)?;
547 for m in modifiers {
548 usage(*m)?;
549 }
550 }
551 let (Some(board), Some(keyboard)) = (self.board.as_mut(), self.keyboard.as_mut()) else {
552 return Err(anyhow!("no keyboard attached"));
553 };
554 for (key, modifiers) in combos {
555 crate::device::ax::trace(|| format!("keyboard: combo {key:#x}"));
556 for m in modifiers {
557 keyboard
558 .main_keyboard_key_down(board, usage(*m)?)
559 .await
560 .map_err(|e| anyhow!("modifier down: {e}"))?;
561 tokio::time::sleep(Duration::from_millis(20)).await;
562 }
563 keyboard
564 .main_keyboard_key_down(board, usage(*key)?)
565 .await
566 .map_err(|e| anyhow!("key down: {e}"))?;
567 tokio::time::sleep(Duration::from_millis(40)).await;
568 keyboard
570 .set_main_keyboard_usages(board, std::iter::empty())
571 .await
572 .map_err(|e| anyhow!("key up: {e}"))?;
573 tokio::time::sleep(Duration::from_millis(80)).await;
574 }
575 Ok(())
576 }
577
578 pub async fn keyboard_detach(&mut self) -> Result<()> {
581 let Some(mut board) = self.board.take() else {
582 return Ok(());
583 };
584 let Some(keyboard) = self.keyboard.as_mut() else {
585 return Ok(());
586 };
587 crate::device::ax::trace(|| "keyboard: removing".into());
588 let removed = keyboard.remove_main_keyboard(&mut board).await;
589 crate::device::ax::trace(|| "keyboard: removed".into());
590 removed.map_err(|e| anyhow!("remove keyboard: {e}"))
591 }
592
593 pub async fn keys(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
595 self.keyboard_attach().await?;
596 let pressed = self.keyboard_press(combos).await;
597 let detached = self.keyboard_detach().await;
599 pressed?;
600 detached
601 }
602
603 pub async fn key(&mut self, usage: u64, modifiers: &[u64]) -> Result<()> {
605 self.keys(&[(usage, modifiers.to_vec())]).await
606 }
607
608 pub async fn close(mut self) {
609 let _ = self.keyboard_detach().await;
610 self.settle_previous().await;
611 let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
612 }
613
614 pub async fn abandon(mut self) {
617 let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
618 }
619}