use std::{
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
time::{Duration, Instant},
};
use anyhow::{Result, anyhow};
use idevice::{
ReadWrite, RsdService,
core_device::{
CallInfoBlob, DisplayServiceClient, build_screen_audio_offer, build_screen_video_offer,
build_start_audio_parameters, build_start_video_parameters,
hid::{
ButtonState, IndigoHidClient, KeyboardUsage, MainKeyboardService,
TOUCHSCREEN_STATE_CONTACT, TOUCHSCREEN_STATE_RELEASE, TouchscreenContact,
UniversalHidServiceClient,
},
},
rsd::RsdHandshake,
};
use crate::{
device::pinch,
tcp::handle::{AdapterHandle, UdpSocketHandle},
};
const CLIENT_SUPPORTED_FEATURES: u64 = 140;
const DEVICES_READY: Duration = Duration::from_secs(5);
const WARMUP: Duration = Duration::from_millis(300);
const SETTLE: Duration = Duration::from_millis(300);
const TOUCH_GAP: Duration = Duration::from_millis(150);
const FIRST_PACKET: Duration = Duration::from_secs(3);
const KEYBOARD_ATTACH: Duration = Duration::from_millis(700);
fn warmup() -> Duration {
std::env::var("IPH_INPUT_WARMUP_MS")
.ok()
.and_then(|v| v.parse().ok())
.map(Duration::from_millis)
.unwrap_or(WARMUP)
}
async fn devices_ready<R: ReadWrite>(client: &mut UniversalHidServiceClient<R>) -> String {
let began = now_ms();
match tokio::time::timeout(DEVICES_READY, client.list_connected_services()).await {
Ok(Ok(_)) => format!("{} ms", now_ms().saturating_sub(began)),
Ok(Err(e)) => format!("error {e:?}"),
Err(_) => "timed out".to_string(),
}
}
fn now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_millis() as u64)
.unwrap_or(0)
}
struct Drain {
last_packet_ms: Arc<AtomicU64>,
first_packet_ms: Vec<Arc<AtomicU64>>,
tasks: Vec<tokio::task::JoinHandle<()>>,
}
impl Drain {
fn start(sockets: Vec<UdpSocketHandle>) -> Self {
let last_packet_ms = Arc::new(AtomicU64::new(0));
let first_packet_ms: Vec<Arc<AtomicU64>> =
sockets.iter().map(|_| Arc::new(AtomicU64::new(0))).collect();
let counts: Vec<Arc<(AtomicU64, AtomicU64)>> =
sockets.iter().map(|_| Arc::new((AtomicU64::new(0), AtomicU64::new(0)))).collect();
let mut tasks: Vec<tokio::task::JoinHandle<()>> = sockets
.into_iter()
.zip(first_packet_ms.iter().cloned())
.zip(counts.iter().cloned())
.map(|((s, first), count)| {
let last = last_packet_ms.clone();
tokio::spawn(async move {
while let Ok(packet) = s.recv().await {
let now = now_ms();
last.store(now, Ordering::Relaxed);
let _ = first.compare_exchange(0, now, Ordering::Relaxed, Ordering::Relaxed);
count.0.fetch_add(packet.data.len() as u64, Ordering::Relaxed);
count.1.fetch_add(1, Ordering::Relaxed);
}
})
})
.collect();
if std::env::var("IPH_TRACE").is_ok_and(|v| v == "1") {
tasks.push(tokio::spawn(async move {
let mut before: Vec<(u64, u64)> = counts.iter().map(|_| (0, 0)).collect();
loop {
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
let now: Vec<(u64, u64)> = counts
.iter()
.map(|c| (c.0.load(Ordering::Relaxed), c.1.load(Ordering::Relaxed)))
.collect();
let rates: Vec<String> = now
.iter()
.zip(&before)
.map(|(n, b)| format!("{} B/{} pkts", n.0 - b.0, n.1 - b.1))
.collect();
crate::device::ax::trace(|| format!("media in the last second: {}", rates.join(", ")));
before = now;
}
}));
}
Self {
last_packet_ms,
first_packet_ms,
tasks,
}
}
fn flowing(&self) -> bool {
self.first_packet_ms.iter().all(|f| f.load(Ordering::Relaxed) > 0)
}
}
impl Drop for Drain {
fn drop(&mut self) {
for t in &self.tasks {
t.abort();
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Motion {
Flick,
Place,
}
pub struct InputSession {
display: DisplayServiceClient<Box<dyn ReadWrite>>,
hid: UniversalHidServiceClient<Box<dyn ReadWrite>>,
keyboard: Option<UniversalHidServiceClient<Box<dyn ReadWrite>>>,
board: Option<MainKeyboardService>,
indigo: Option<IndigoHidClient<Box<dyn ReadWrite>>>,
adapter: AdapterHandle,
rsd: RsdHandshake,
drain: Drain,
pub started: Instant,
last_release: Option<Instant>,
reports: u64,
pair_held: Option<[TouchscreenContact; 2]>,
}
pub fn normalize(v: f64, extent: f64) -> u16 {
if extent <= 0.0 {
return 0;
}
((v / extent) * 65535.0).round().clamp(0.0, 65535.0) as u16
}
impl InputSession {
pub async fn open(adapter: &mut AdapterHandle, rsd: &mut RsdHandshake, video: bool) -> Result<Self> {
let began = Instant::now();
let began_ms = now_ms();
let mut display = DisplayServiceClient::connect_rsd(adapter, rsd)
.await
.map_err(|e| anyhow!("display service: {e:?}"))?;
let audio_udp = adapter.bind_udp(0).await?;
let video_udp = adapter.bind_udp(0).await?;
let receiver_ip = adapter.host_ip().to_string();
let sender_ip = adapter.peer_ip().to_string();
let call_info = CallInfoBlob {
call_id: 0,
client_version: 1,
device_type: "Mac17,7".into(),
framework_version: "2205.3.1".into(),
os_version: "25F71".into(),
device_name: None,
audio_device_uid: None,
};
let session_id = uuid::Uuid::new_v4();
let audio_call = uuid::Uuid::new_v4().to_string().to_uppercase();
let audio_offer = build_screen_audio_offer(&audio_call, &call_info)
.map_err(|e| anyhow!("audio offer: {e:?}"))?;
let audio = display
.start_media_stream(build_start_audio_parameters(
&receiver_ip,
audio_udp.local_port(),
&sender_ip,
50000,
audio_offer,
CLIENT_SUPPORTED_FEATURES,
session_id,
))
.await;
let mut streams = Vec::new();
let mut video = video;
match audio {
Ok(_) => streams.push(audio_udp),
Err(e) if format!("{e:?}").contains("Integer(9022)") => video = true,
Err(e) => return Err(anyhow!("start audio stream: {e:?}")),
}
if video {
let video_call = uuid::Uuid::new_v4().to_string().to_uppercase();
let ssrc = uuid::Uuid::new_v4().as_u128() as u32;
let video_offer = build_screen_video_offer(&video_call, &call_info, ssrc)
.map_err(|e| anyhow!("video offer: {e:?}"))?;
display
.start_media_stream(build_start_video_parameters(
&receiver_ip,
video_udp.local_port(),
&sender_ip,
50001,
video_offer,
CLIENT_SUPPORTED_FEATURES,
1,
session_id,
))
.await
.map_err(|e| anyhow!("start video stream: {e:?}"))?;
streams.push(video_udp);
}
let drain = Drain::start(streams);
let mut hid = UniversalHidServiceClient::connect_rsd(adapter, rsd)
.await
.map_err(|e| anyhow!("hid service: {e:?}"))?;
let hid_ready = devices_ready(&mut hid).await;
let prime = std::env::var("IPH_INPUT_PRIME").map(|v| v != "0").unwrap_or(true);
if prime {
let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
}
tokio::time::sleep(warmup()).await;
let waiting = Instant::now();
while !drain.flowing() && waiting.elapsed() < FIRST_PACKET {
tokio::time::sleep(Duration::from_millis(50)).await;
}
crate::device::ax::trace(|| {
let firsts: Vec<String> = drain
.first_packet_ms
.iter()
.map(|f| match f.load(Ordering::Relaxed) {
0 => "none".to_string(),
at => format!("{} ms", at.saturating_sub(began_ms)),
})
.collect();
format!(
"input: session open after {} ms, devices ready at {hid_ready}, first audio/video packet at {}",
began.elapsed().as_millis(),
firsts.join(" / ")
)
});
if prime {
let _ = hid.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, 65535, 65535, None).await;
tokio::time::sleep(Duration::from_millis(60)).await;
}
Ok(Self {
display,
hid,
keyboard: None,
board: None,
indigo: None,
adapter: adapter.clone(),
rsd: rsd.clone(),
drain,
started: Instant::now(),
last_release: None,
reports: 0,
pair_held: None,
})
}
pub fn media_idle_ms(&self) -> Option<u64> {
let last = self.drain.last_packet_ms.load(Ordering::Relaxed);
(last > 0).then(|| now_ms().saturating_sub(last))
}
async fn wait_since_release(&mut self, gap: Duration) {
if let Some(t) = self.last_release {
let since = t.elapsed();
if since < gap {
tokio::time::sleep(gap - since).await;
}
}
}
async fn settle_previous(&mut self) {
self.wait_since_release(SETTLE).await;
}
async fn gap(&mut self) {
self.wait_since_release(TOUCH_GAP).await;
}
pub fn reports_sent(&self) -> u64 {
self.reports
}
async fn contact(&mut self, x: u16, y: u16) -> Result<()> {
crate::device::ax::trace(|| format!("contact {x},{y}"));
self.hid
.send_touchscreen(TOUCHSCREEN_STATE_CONTACT, x, y, None)
.await
.map_err(|e| anyhow!("touch contact: {e:?}"))?;
self.reports += 1;
Ok(())
}
async fn release(&mut self, x: u16, y: u16) -> Result<()> {
let r = self
.hid
.send_touchscreen(TOUCHSCREEN_STATE_RELEASE, x, y, None)
.await
.map_err(|e| anyhow!("touch release: {e:?}"));
crate::device::ax::trace(|| "touch released".into());
self.last_release = Some(Instant::now());
if r.is_ok() {
self.reports += 1;
}
r
}
pub async fn tap(&mut self, x: u16, y: u16, hold: Duration) -> Result<()> {
self.gap().await;
crate::device::ax::trace(|| format!("tap down {x},{y}"));
self.contact(x, y).await?;
tokio::time::sleep(hold).await;
self.release(x, y).await
}
pub async fn double_tap(&mut self, x: u16, y: u16) -> Result<()> {
self.gap().await;
for i in 0..2 {
self.contact(x, y).await?;
tokio::time::sleep(Duration::from_millis(50)).await;
self.release(x, y).await?;
if i == 0 {
tokio::time::sleep(Duration::from_millis(80)).await;
}
}
Ok(())
}
pub async fn drag(
&mut self,
from: (u16, u16),
to: (u16, u16),
duration: Duration,
motion: Motion,
) -> Result<()> {
self.gap().await;
let steps = (duration.as_millis() / 16).clamp(4, 240) as u32;
let start = Instant::now();
crate::device::ax::trace(|| format!("drag start {from:?} -> {to:?}"));
let mut sent_any = false;
let mut last = from;
let res: Result<()> = async {
for i in 0..=steps {
let t = i as f64 / steps as f64;
let eased = match motion {
Motion::Flick => t,
Motion::Place => 1.0 - (1.0 - t).powi(3),
};
let x = (from.0 as f64 + (to.0 as f64 - from.0 as f64) * eased).round() as u16;
let y = (from.1 as f64 + (to.1 as f64 - from.1 as f64) * eased).round() as u16;
self.contact(x, y).await?;
sent_any = true;
last = (x, y);
let due = duration.mul_f64(t);
let elapsed = start.elapsed();
if due > elapsed {
tokio::time::sleep(due - elapsed).await;
}
}
if motion == Motion::Place {
for _ in 0..4 {
tokio::time::sleep(Duration::from_millis(30)).await;
self.contact(to.0, to.1).await?;
}
}
Ok(())
}
.await;
if sent_any {
let lifted = self.release(last.0, last.1).await;
res?;
lifted?;
} else {
res?;
}
Ok(())
}
pub async fn pinch(&mut self, plan: &pinch::Plan) -> Result<()> {
self.gap().await;
crate::device::ax::trace(|| format!("pinch {} frames, span {:?}", plan.frames.len(), plan.span));
let (hid, reports) = (&mut self.hid, &mut self.reports);
let played = pinch::play(plan, &mut self.pair_held, async |contacts: &[TouchscreenContact]| -> Result<()> {
hid.send_multitouch(contacts, None)
.await
.map_err(|e| anyhow!("touch pinch: {e:?}"))?;
*reports += 1;
Ok(())
})
.await;
self.last_release = Some(Instant::now());
played
}
async fn lift_held(&mut self) {
let Some(pair) = self.pair_held.take() else {
return;
};
let lifted = pinch::lifted(&pair);
let _ = tokio::time::timeout(Duration::from_secs(2), self.hid.send_multitouch(&lifted, None)).await;
self.last_release = Some(Instant::now());
}
pub async fn button(&mut self, usage_page: u64, usage: u64, hold: Duration) -> Result<()> {
if self.indigo.is_none() {
let indigo = IndigoHidClient::connect_rsd(&mut self.adapter, &mut self.rsd)
.await
.map_err(|e| anyhow!("indigo HID service unavailable: {e:?}"))?;
self.indigo = Some(indigo);
let ready = devices_ready(&mut self.hid).await;
tokio::time::sleep(WARMUP).await;
crate::device::ax::trace(|| format!("input: buttons connected, devices ready at {ready}"));
}
let Some(indigo) = self.indigo.as_mut() else {
unreachable!()
};
indigo
.send_button(usage_page, usage, ButtonState::Down)
.await
.map_err(|e| anyhow!("button down: {e:?}"))?;
self.reports += 1;
tokio::time::sleep(hold).await;
indigo
.send_button(usage_page, usage, ButtonState::Up)
.await
.map_err(|e| anyhow!("button up: {e:?}"))?;
self.last_release = Some(Instant::now());
Ok(())
}
pub async fn keyboard_attach(&mut self) -> Result<()> {
if self.board.is_some() {
return Ok(());
}
if self.keyboard.is_none() {
let mut keyboard = UniversalHidServiceClient::connect_rsd(&mut self.adapter, &mut self.rsd)
.await
.map_err(|e| anyhow!("hid service (keyboard): {e:?}"))?;
let ready = devices_ready(&mut keyboard).await;
tokio::time::sleep(WARMUP).await;
crate::device::ax::trace(|| format!("keyboard: connected, devices ready at {ready}"));
self.keyboard = Some(keyboard);
}
let Some(keyboard) = self.keyboard.as_mut() else {
unreachable!()
};
crate::device::ax::trace(|| "keyboard: registering".into());
let board = keyboard
.create_main_keyboard()
.await
.map_err(|e| anyhow!("register keyboard: {e}"))?;
self.board = Some(board);
tokio::time::sleep(KEYBOARD_ATTACH).await;
crate::device::ax::trace(|| "keyboard: registered".into());
Ok(())
}
pub async fn keyboard_press(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
let usage = |raw: u64| {
KeyboardUsage::new(raw as u16).map_err(|e| anyhow!("key usage {raw:#x}: {e}"))
};
for (key, modifiers) in combos {
usage(*key)?;
for m in modifiers {
usage(*m)?;
}
}
let (Some(board), Some(keyboard)) = (self.board.as_mut(), self.keyboard.as_mut()) else {
return Err(anyhow!("no keyboard attached"));
};
for (key, modifiers) in combos {
crate::device::ax::trace(|| format!("keyboard: combo {key:#x}"));
for m in modifiers {
keyboard
.main_keyboard_key_down(board, usage(*m)?)
.await
.map_err(|e| anyhow!("modifier down: {e}"))?;
tokio::time::sleep(Duration::from_millis(20)).await;
}
keyboard
.main_keyboard_key_down(board, usage(*key)?)
.await
.map_err(|e| anyhow!("key down: {e}"))?;
tokio::time::sleep(Duration::from_millis(40)).await;
keyboard
.set_main_keyboard_usages(board, std::iter::empty())
.await
.map_err(|e| anyhow!("key up: {e}"))?;
tokio::time::sleep(Duration::from_millis(80)).await;
}
Ok(())
}
pub async fn keyboard_detach(&mut self) -> Result<()> {
let Some(mut board) = self.board.take() else {
return Ok(());
};
let Some(keyboard) = self.keyboard.as_mut() else {
return Ok(());
};
crate::device::ax::trace(|| "keyboard: removing".into());
let removed = keyboard.remove_main_keyboard(&mut board).await;
crate::device::ax::trace(|| "keyboard: removed".into());
removed.map_err(|e| anyhow!("remove keyboard: {e}"))
}
pub async fn keys(&mut self, combos: &[(u64, Vec<u64>)]) -> Result<()> {
self.keyboard_attach().await?;
let pressed = self.keyboard_press(combos).await;
let detached = self.keyboard_detach().await;
pressed?;
detached
}
pub async fn key(&mut self, usage: u64, modifiers: &[u64]) -> Result<()> {
self.keys(&[(usage, modifiers.to_vec())]).await
}
pub async fn close(mut self) {
self.lift_held().await;
let _ = self.keyboard_detach().await;
self.settle_previous().await;
let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
}
pub async fn abandon(mut self) {
self.lift_held().await;
let _ = tokio::time::timeout(Duration::from_secs(2), self.display.stop_media_stream()).await;
}
}