Skip to main content

iphone_cli/device/
ax.rs

1//! Subscription to the phone's accessibility tree.
2//!
3//! One DTX connection to `…axAuditDaemon.remoteAXService`.
4//!
5//! What the phone does (iOS 27.0.1, measured 2026-11-10):
6//! - `clientNeedsAccessibility:` (true) asks for a dump of the whole screen.
7//!   The answer comes in two passes: the first about 1.1 s after the request
8//!   on a still screen (up to several seconds while something animates), the
9//!   second about 0.5 s later.
10//! - The 1.1 s is a quiet period that every new request restarts. Requests
11//!   sent every second never get an answer. So: one request, then wait.
12//! - After a navigation in Settings the phone sent nothing on its own; a
13//!   fresh reading is a new request on the same connection. Some apps do make
14//!   the phone send dumps unasked (Calculator on every key, Notes when the
15//!   keyboard comes or goes). Those lag behind what triggered them, so one
16//!   can arrive after a later request and look like its answer.
17
18use std::{
19    sync::{Arc, Mutex},
20    time::{Duration, Instant},
21};
22
23use anyhow::{Result, anyhow};
24use idevice::{
25    dvt::{message::AuxValue, remote_server::RemoteServerClient},
26    rsd::RsdHandshake,
27};
28use tokio::sync::{mpsc, watch};
29
30use crate::ax::{
31    archive::Ka,
32    dump::{Answer, Dump, DumpKind, ObjKey, Payload},
33    request,
34};
35use crate::tcp::handle::AdapterHandle;
36
37pub const SERVICE: &str = "com.apple.accessibility.axAuditDaemon.remoteAXService";
38/// The phone never answers a request sooner than this (about 1.1 s typical).
39pub const ANSWER_AT_LEAST: Duration = Duration::from_millis(700);
40const SUBSCRIBE: &str = "clientNeedsAccessibility:";
41const PUSH: &str = "processDataFromRemoteDevice:";
42const ASK: &str = "processDataFromHost:";
43
44#[derive(Default)]
45pub struct AxState {
46    /// Number of first-pass dumps received so far.
47    pub seq: u64,
48    pub initial: Option<Arc<Dump>>,
49    /// When the latest first-pass dump arrived.
50    pub initial_at: Option<Instant>,
51    /// Arrival times of the most recent first-pass dumps.
52    pub arrivals: std::collections::VecDeque<Instant>,
53    /// When the last request went out, and whether its answer is still due.
54    pub asked_at: Option<Instant>,
55    pub answer_due: bool,
56    /// When the phone last sent a first-pass dump nobody asked for.
57    pub unasked_at: Option<Instant>,
58    /// Payloads the parser gave up on.
59    pub unparsed: u64,
60    /// `seq` of the reading that `full` belongs to.
61    pub full_seq: u64,
62    /// First pass merged with the second pass (identifiers, hints).
63    pub full: Option<Arc<Dump>>,
64    /// Dumps that arrived without any element (seen during animations).
65    pub empty: u64,
66    pub closed: bool,
67    /// Answers to `ask`, newest last, until the asker takes them.
68    pub answers: std::collections::VecDeque<(Instant, Answer)>,
69}
70
71enum Cmd {
72    Subscribe,
73    Ask(Vec<u8>),
74}
75
76/// `IPH_TRACE=1` logs every request and dump with a millisecond clock, to the
77/// daemon's log.
78pub fn trace(msg: impl FnOnce() -> String) {
79    use std::sync::OnceLock;
80    static ON: OnceLock<bool> = OnceLock::new();
81    static START: OnceLock<Instant> = OnceLock::new();
82    if *ON.get_or_init(|| std::env::var("IPH_TRACE").is_ok_and(|v| v == "1")) {
83        let t = START.get_or_init(Instant::now).elapsed().as_millis();
84        eprintln!("[trace {t:>8}] {}", msg());
85    }
86}
87
88pub struct AxSub {
89    cmd: mpsc::UnboundedSender<Cmd>,
90    state: Arc<Mutex<AxState>>,
91    changed: watch::Receiver<u64>,
92    task: tokio::task::JoinHandle<()>,
93    /// When the connection's first request went out.
94    pub opened: Instant,
95}
96
97impl Drop for AxSub {
98    fn drop(&mut self) {
99        self.task.abort();
100    }
101}
102
103impl AxSub {
104    pub async fn open(adapter: &mut AdapterHandle, rsd: &RsdHandshake) -> Result<Self> {
105        let port = rsd
106            .services
107            .get(SERVICE)
108            .ok_or_else(|| anyhow!("the phone does not offer {SERVICE}"))?
109            .port;
110        let stream = adapter.connect(port).await?;
111        let mut dtx = RemoteServerClient::new(stream);
112        subscribe(&mut dtx).await?;
113        let opened = Instant::now();
114
115        let state = Arc::new(Mutex::new(AxState {
116            asked_at: Some(opened),
117            answer_due: true,
118            ..Default::default()
119        }));
120        let (cmd_tx, mut cmd_rx) = mpsc::unbounded_channel::<Cmd>();
121        let (changed_tx, changed_rx) = watch::channel(0u64);
122        let task_state = state.clone();
123        let task = tokio::spawn(async move {
124            let mut ticks = 0u64;
125            loop {
126                tokio::select! {
127                    cmd = cmd_rx.recv() => match cmd {
128                        Some(Cmd::Subscribe) => {
129                            trace(|| "ax request".into());
130                            if subscribe(&mut dtx).await.is_err() {
131                                break;
132                            }
133                        }
134                        Some(Cmd::Ask(bytes)) => {
135                            trace(|| "ax ask".into());
136                            let sent = dtx
137                                .call_method(0, Some(ASK), Some(vec![AuxValue::Array(bytes)]), false)
138                                .await;
139                            if sent.is_err() {
140                                break;
141                            }
142                        }
143                        None => break,
144                    },
145                    msg = dtx.read_message(0) => {
146                        let Ok(msg) = msg else { break };
147                        let is_push = matches!(&msg.data, Some(plist::Value::String(s)) if s == PUSH);
148                        if !is_push {
149                            continue;
150                        }
151                        let Some(AuxValue::Array(bytes)) =
152                            msg.aux.and_then(|a| a.values.into_iter().next())
153                        else {
154                            continue;
155                        };
156                        let arrived = Instant::now();
157                        debug_keep(&bytes, false);
158                        let size = bytes.len();
159                        // Off the async threads: a payload that takes long to
160                        // unfold must not stall the tunnel and the timers.
161                        let parsed = tokio::task::spawn_blocking(move || {
162                            let r = Payload::parse(&bytes);
163                            (r, bytes)
164                        })
165                        .await;
166                        let dump = match parsed {
167                            Ok((Ok(Payload::Dump(dump)), _)) => dump,
168                            Ok((Ok(Payload::Notification(n)), _)) => {
169                                trace(|| format!("ax notification {n} bytes={size}"));
170                                continue;
171                            }
172                            Ok((Ok(Payload::Answer(a)), _)) => {
173                                trace(|| format!("ax answer attribute={} bytes={size}", a.attribute));
174                                let mut s = task_state.lock().unwrap();
175                                s.answers.push_back((arrived, a));
176                                while s.answers.len() > 32 {
177                                    s.answers.pop_front();
178                                }
179                                drop(s);
180                                ticks += 1;
181                                let _ = changed_tx.send(ticks);
182                                continue;
183                            }
184                            Ok((Err(e), bytes)) => {
185                                debug_keep(&bytes, true);
186                                eprintln!("[{}] accessibility payload of {size} bytes not understood: {e:#}", crate::clock::now_utc());
187                                task_state.lock().unwrap().unparsed += 1;
188                                ticks += 1;
189                                let _ = changed_tx.send(ticks);
190                                continue;
191                            }
192                            Err(_) => continue,
193                        };
194                        trace(|| format!("ax dump {:?} elements={} bytes={size}", dump.kind, dump.elements.len()));
195                        if accept(&task_state, dump, arrived) {
196                            ticks += 1;
197                            let _ = changed_tx.send(ticks);
198                        }
199                    }
200                }
201            }
202            task_state.lock().unwrap().closed = true;
203            ticks += 1;
204            let _ = changed_tx.send(ticks);
205        });
206
207        Ok(Self {
208            cmd: cmd_tx,
209            state,
210            changed: changed_rx,
211            task,
212            opened,
213        })
214    }
215
216    pub fn is_closed(&self) -> bool {
217        self.state.lock().unwrap().closed
218    }
219
220    /// Ask the phone for a new dump. Returns when the request was made.
221    ///
222    /// The phone answers a request about 1.1 s after the *last* one it got:
223    /// every new request restarts that wait, so asking again while an answer
224    /// is on its way only delays it.
225    pub fn request(&self) -> Result<Instant> {
226        self.cmd
227            .send(Cmd::Subscribe)
228            .map_err(|_| anyhow!("accessibility connection closed"))?;
229        let now = Instant::now();
230        let mut s = self.state.lock().unwrap();
231        s.asked_at = Some(now);
232        s.answer_due = true;
233        Ok(now)
234    }
235
236    /// True when the phone sent a dump nobody asked for at or after `since`.
237    pub fn unasked_since(&self, since: Instant) -> bool {
238        self.state.lock().unwrap().unasked_at.is_some_and(|t| t >= since)
239    }
240
241    pub fn unparsed_count(&self) -> u64 {
242        self.state.lock().unwrap().unparsed
243    }
244
245    /// Wait for a first-pass dump that arrived at or after `earliest`.
246    pub async fn wait_arrived(&mut self, earliest: Instant, timeout: Duration) -> Option<(u64, Arc<Dump>)> {
247        let deadline = Instant::now() + timeout;
248        loop {
249            {
250                let s = self.state.lock().unwrap();
251                if s.initial_at.is_some_and(|at| at >= earliest)
252                    && let Some(d) = s.initial.clone() {
253                        return Some((s.seq, d));
254                    }
255                if s.closed {
256                    return None;
257                }
258            }
259            let left = deadline.checked_duration_since(Instant::now())?;
260            match tokio::time::timeout(left, self.changed.changed()).await {
261                Ok(Ok(())) => {}
262                _ => return None,
263            }
264        }
265    }
266
267    /// Ask for one attribute of one element. The phone answers in tens of
268    /// milliseconds and, unlike a dump, does not wait for the screen to be
269    /// still. `Some(Ka::Null)` means the element has no such attribute (or is
270    /// gone); `None` that no answer came within `timeout`.
271    pub async fn ask(&mut self, key: ObjKey, attribute: i64, timeout: Duration) -> Option<Ka> {
272        let asked = Instant::now();
273        self.cmd.send(Cmd::Ask(request::attribute(key, attribute))).ok()?;
274        let deadline = asked + timeout;
275        loop {
276            {
277                let mut s = self.state.lock().unwrap();
278                let hit = s.answers.iter().position(|(at, a)| {
279                    *at >= asked && a.attribute == attribute && a.key == Some(key)
280                });
281                if let Some(i) = hit {
282                    return s.answers.remove(i).map(|(_, a)| a.value);
283                }
284                if s.closed {
285                    return None;
286                }
287            }
288            let left = deadline.checked_duration_since(Instant::now())?;
289            match tokio::time::timeout(left, self.changed.changed()).await {
290                Ok(Ok(())) => {}
291                _ => return None,
292            }
293        }
294    }
295
296    pub fn seq(&self) -> u64 {
297        self.state.lock().unwrap().seq
298    }
299
300    pub fn empty_count(&self) -> u64 {
301        self.state.lock().unwrap().empty
302    }
303
304    /// Latest first-pass dump with its sequence number and age.
305    pub fn latest(&self) -> Option<(u64, Arc<Dump>, Duration)> {
306        let s = self.state.lock().unwrap();
307        Some((s.seq, s.initial.clone()?, s.initial_at?.elapsed()))
308    }
309
310    /// When the latest first-pass dump arrived.
311    pub fn latest_arrival(&self) -> Option<Instant> {
312        self.state.lock().unwrap().initial_at
313    }
314
315    /// How many first-pass dumps arrived at or after `since`.
316    pub fn initials_since(&self, since: Instant) -> usize {
317        self.state.lock().unwrap().arrivals.iter().filter(|t| **t >= since).count()
318    }
319
320    /// Latest dump with second-pass attributes, if it belongs to reading `seq`.
321    pub fn full(&self, seq: u64) -> Option<Arc<Dump>> {
322        let s = self.state.lock().unwrap();
323        (s.full_seq == seq).then(|| s.full.clone()).flatten()
324    }
325
326    /// Wait for the second pass of reading `seq`.
327    pub async fn wait_full(&mut self, seq: u64, timeout: Duration) -> Option<Arc<Dump>> {
328        let deadline = Instant::now() + timeout;
329        loop {
330            {
331                let s = self.state.lock().unwrap();
332                if s.full_seq == seq
333                    && let Some(d) = s.full.clone() {
334                        return Some(d);
335                    }
336                // A newer reading replaced this one; its second pass will not come.
337                if s.seq > seq || s.closed {
338                    return None;
339                }
340            }
341            let left = deadline.checked_duration_since(Instant::now())?;
342            match tokio::time::timeout(left, self.changed.changed()).await {
343                Ok(Ok(())) => {}
344                _ => return None,
345            }
346        }
347    }
348}
349
350/// `IPH_DUMP_DIR=<dir>` keeps the last few raw payloads on disk before they
351/// are parsed, to study a payload that the parser chokes on. The payloads
352/// hold whatever is on the phone's screen: use it on test screens only and
353/// delete the files afterwards.
354fn debug_keep(bytes: &[u8], failed: bool) {
355    use std::sync::atomic::{AtomicU64, Ordering};
356    static N: AtomicU64 = AtomicU64::new(0);
357    let Ok(dir) = std::env::var("IPH_DUMP_DIR") else {
358        return;
359    };
360    if failed {
361        let n = N.load(Ordering::Relaxed);
362        let _ = std::fs::write(format!("{dir}/failed-{n}.raw.bin"), bytes);
363        return;
364    }
365    let n = N.fetch_add(1, Ordering::Relaxed);
366    let _ = std::fs::write(format!("{dir}/payload-{}.raw.bin", n % 8), bytes);
367    trace(|| format!("ax payload #{n} bytes={} kept as payload-{}", bytes.len(), n % 8));
368}
369
370async fn subscribe(dtx: &mut RemoteServerClient<Box<dyn idevice::ReadWrite>>) -> Result<()> {
371    dtx.call_method(
372        0,
373        Some(SUBSCRIBE),
374        Some(vec![AuxValue::archived_value(true)]),
375        false,
376    )
377    .await
378    .map_err(|e| anyhow!("{SUBSCRIBE} failed: {e:?}"))
379}
380
381/// Store a dump. Returns false for dumps that carry nothing.
382fn accept(state: &Arc<Mutex<AxState>>, dump: Dump, arrived: Instant) -> bool {
383    if dump.elements.is_empty() || dump.applications.is_empty() {
384        state.lock().unwrap().empty += 1;
385        return false;
386    }
387    match dump.kind {
388        DumpKind::Initial => {
389            let mut s = state.lock().unwrap();
390            s.seq += 1;
391            s.initial = Some(Arc::new(dump));
392            s.initial_at = Some(arrived);
393            // An answer needs about a second. A dump that comes while none is
394            // due, or sooner than an answer can, was sent unasked: the app is
395            // one that makes the phone do that.
396            let answers = s.answer_due
397                && s.asked_at.is_some_and(|at| arrived >= at + ANSWER_AT_LEAST);
398            if answers {
399                s.answer_due = false;
400            } else {
401                s.unasked_at = Some(arrived);
402            }
403            if s.arrivals.len() >= 32 {
404                s.arrivals.pop_front();
405            }
406            s.arrivals.push_back(arrived);
407            true
408        }
409        DumpKind::Additional => {
410            let (seq, initial) = {
411                let s = state.lock().unwrap();
412                match s.initial.clone() {
413                    Some(i) => (s.seq, i),
414                    None => return false,
415                }
416            };
417            // The second pass belongs to the newest first pass only if it
418            // describes the same elements.
419            if !initial.same_elements(&dump) {
420                return false;
421            }
422            // Merge without holding the lock; readers must never wait on it.
423            let mut merged = (*initial).clone();
424            merged.absorb(&dump);
425            let mut s = state.lock().unwrap();
426            if s.seq != seq {
427                return false;
428            }
429            s.full = Some(Arc::new(merged));
430            s.full_seq = seq;
431            true
432        }
433        DumpKind::Other(_) => false,
434    }
435}